diff --git a/apps/server/drizzle/0020_bizarre_otto_octavius.sql b/apps/server/drizzle/0020_bizarre_otto_octavius.sql
new file mode 100644
index 000000000..e29f57e7b
--- /dev/null
+++ b/apps/server/drizzle/0020_bizarre_otto_octavius.sql
@@ -0,0 +1,156 @@
+ALTER TYPE "public"."job_status" ADD VALUE 'cancelled';--> statement-breakpoint
+ALTER TABLE "ccip_embeddings" ADD COLUMN "input_revision" text;--> statement-breakpoint
+ALTER TABLE "ccip_embeddings" ADD COLUMN "preprocessing_profile" text DEFAULT 'dghs-imgutils-rs/full-image-default/v1' NOT NULL;--> statement-breakpoint
+ALTER TABLE "ccip_embeddings" DROP CONSTRAINT "uq_ccip_embeddings_region_model_version";--> statement-breakpoint
+ALTER TABLE "ccip_embeddings" ADD CONSTRAINT "uq_ccip_embeddings_region_model_version" UNIQUE("region_id","model","embedding_version","preprocessing_profile");--> statement-breakpoint
+DROP INDEX IF EXISTS "idx_ccip_embeddings_embedding_cosine";--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "queue_name" text;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "target_id" text;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "input_revision" text;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "dedupe_key" text;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "concurrency_key" text;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "available_at" timestamp DEFAULT now() NOT NULL;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "attempt_count" integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "max_attempts" integer DEFAULT 5 NOT NULL;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "lease_duration_ms" integer DEFAULT 300000 NOT NULL;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "claim_token" uuid;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "claimed_by" text;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "claimed_at" timestamp;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "heartbeat_at" timestamp;--> statement-breakpoint
+ALTER TABLE "jobs" ADD COLUMN "error_code" text;--> statement-breakpoint
+ALTER TABLE "lancedb_sync_dirty" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "source_width" integer;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "source_height" integer;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "source_revision" text;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "region_revision" text;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "label" text;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "manual_reason" text;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "detection_key" text;--> statement-breakpoint
+ALTER TABLE "media_regions" ADD COLUMN "detector_model" text;--> statement-breakpoint
+ALTER TABLE "media_relations" ADD COLUMN "source_region_id" uuid;--> statement-breakpoint
+ALTER TABLE "media_relations" ADD COLUMN "derivation_key" text;--> statement-breakpoint
+UPDATE "jobs"
+SET
+ "queue_name" = CASE
+ WHEN "type" IN ('auto_tagging', 'extract_ccip_vector') THEN 'ai'
+ ELSE 'default'
+ END,
+ "available_at" = "created_at";--> statement-breakpoint
+DO $$
+BEGIN
+ IF EXISTS (
+ SELECT 1
+ FROM "media_regions" AS region
+ INNER JOIN "media" AS media ON media."id" = region."media_id"
+ WHERE media."width" <= 0 OR media."height" <= 0
+ ) THEN
+ RAISE EXCEPTION 'Cannot migrate media_regions: source media dimensions must be positive';
+ END IF;
+END
+$$;--> statement-breakpoint
+UPDATE "media_regions" AS region
+SET
+ "source_width" = media."width",
+ "source_height" = media."height",
+ "source_revision" = encode(
+ sha256(
+ convert_to(
+ concat(
+ '{"version":1,"mediaId":', to_json(media."id"::text)::text,
+ ',"mediaSourceId":', to_json(media."source_id"::text)::text,
+ ',"modifiedAtMs":', floor(extract(epoch FROM region."source_modified_at") * 1000)::bigint,
+ ',"fileSize":', coalesce(media."file_size"::text, 'null'),
+ ',"width":', media."width",
+ ',"height":', media."height", '}'
+ ),
+ 'UTF8'
+ )
+ ),
+ 'hex'
+ )
+FROM "media" AS media
+WHERE media."id" = region."media_id";--> statement-breakpoint
+UPDATE "media_regions"
+SET "region_revision" = encode(
+ sha256(
+ convert_to(
+ concat(
+ '{"version":1,"sourceRevision":', to_json("source_revision")::text,
+ ',"kind":', to_json("kind"::text)::text,
+ ',"x":', coalesce(to_json("x")::text, 'null'),
+ ',"y":', coalesce(to_json("y")::text, 'null'),
+ ',"width":', coalesce(to_json("width")::text, 'null'),
+ ',"height":', coalesce(to_json("height")::text, 'null'),
+ ',"label":', coalesce(to_json("label")::text, 'null'),
+ ',"detector":', coalesce(to_json("detector")::text, 'null'),
+ ',"detectorModel":', coalesce(to_json("detector_model")::text, 'null'),
+ ',"detectorVersion":', coalesce(to_json("detector_version")::text, 'null'),
+ ',"manualReason":', coalesce(to_json("manual_reason")::text, 'null'), '}'
+ ),
+ 'UTF8'
+ )
+ ),
+ 'hex'
+);--> statement-breakpoint
+WITH embedding_sources AS (
+ SELECT
+ embedding."id",
+ embedding."model",
+ embedding."embedding_version",
+ embedding."preprocessing_profile",
+ encode(
+ sha256(
+ convert_to(
+ concat(
+ '{"version":1,"mediaId":', to_json(media."id"::text)::text,
+ ',"mediaSourceId":', to_json(media."source_id"::text)::text,
+ ',"modifiedAtMs":', floor(extract(epoch FROM embedding."media_modified_at") * 1000)::bigint,
+ ',"fileSize":', coalesce(media."file_size"::text, 'null'),
+ ',"width":', media."width",
+ ',"height":', media."height", '}'
+ ),
+ 'UTF8'
+ )
+ ),
+ 'hex'
+ ) AS source_revision
+ FROM "ccip_embeddings" AS embedding
+ INNER JOIN "media_regions" AS region ON region."id" = embedding."region_id"
+ INNER JOIN "media" AS media ON media."id" = region."media_id"
+)
+UPDATE "ccip_embeddings" AS embedding
+SET "input_revision" = encode(
+ sha256(
+ convert_to(
+ concat(
+ '{"version":1,"sourceRevision":', to_json(source."source_revision")::text,
+ ',"model":', to_json(source."model")::text,
+ ',"embeddingVersion":', source."embedding_version",
+ ',"preprocessingProfile":', to_json(source."preprocessing_profile")::text, '}'
+ ),
+ 'UTF8'
+ )
+ ),
+ 'hex'
+)
+FROM embedding_sources AS source
+WHERE source."id" = embedding."id";--> statement-breakpoint
+ALTER TABLE "media_regions" ALTER COLUMN "source_width" SET NOT NULL;--> statement-breakpoint
+ALTER TABLE "media_regions" ALTER COLUMN "source_height" SET NOT NULL;--> statement-breakpoint
+ALTER TABLE "media_regions" ALTER COLUMN "source_revision" SET NOT NULL;--> statement-breakpoint
+ALTER TABLE "media_regions" ALTER COLUMN "region_revision" SET NOT NULL;--> statement-breakpoint
+ALTER TABLE "ccip_embeddings" ALTER COLUMN "input_revision" SET NOT NULL;--> statement-breakpoint
+ALTER TABLE "media_relations" ADD CONSTRAINT "media_relations_source_region_id_media_regions_id_fk" FOREIGN KEY ("source_region_id") REFERENCES "public"."media_regions"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "idx_jobs_claim" ON "jobs" USING btree ("queue_name","available_at","created_at","id") WHERE "jobs"."status" = 'pending';--> statement-breakpoint
+CREATE INDEX "idx_jobs_stale_lease" ON "jobs" USING btree ("heartbeat_at","claimed_at") WHERE "jobs"."status" = 'in_progress';--> statement-breakpoint
+CREATE INDEX "idx_jobs_parent_status" ON "jobs" USING btree ("parent_id","status");--> statement-breakpoint
+CREATE INDEX "idx_jobs_status_updated" ON "jobs" USING btree ("status","updated_at");--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_jobs_active_dedupe" ON "jobs" USING btree ("dedupe_key") WHERE "jobs"."dedupe_key" IS NOT NULL AND "jobs"."status" IN ('pending', 'in_progress');--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_jobs_running_concurrency" ON "jobs" USING btree ("concurrency_key") WHERE "jobs"."concurrency_key" IS NOT NULL AND "jobs"."status" = 'in_progress';--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_media_regions_detection_key" ON "media_regions" USING btree ("media_id","detection_key") WHERE "media_regions"."detection_key" IS NOT NULL;--> statement-breakpoint
+CREATE INDEX "idx_media_relations_source_region" ON "media_relations" USING btree ("source_region_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_media_relations_derivation_key" ON "media_relations" USING btree ("derivation_key") WHERE "media_relations"."derivation_key" IS NOT NULL;--> statement-breakpoint
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_attempt_count_nonnegative" CHECK ("jobs"."attempt_count" >= 0);--> statement-breakpoint
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_max_attempts_positive" CHECK ("jobs"."max_attempts" > 0);--> statement-breakpoint
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_lease_duration_positive" CHECK ("jobs"."lease_duration_ms" > 0);--> statement-breakpoint
+ALTER TABLE "media_regions" ADD CONSTRAINT "media_regions_source_dimensions_positive" CHECK ("media_regions"."source_width" > 0 AND "media_regions"."source_height" > 0);
diff --git a/apps/server/drizzle/meta/0020_snapshot.json b/apps/server/drizzle/meta/0020_snapshot.json
new file mode 100644
index 000000000..eb20bfd56
--- /dev/null
+++ b/apps/server/drizzle/meta/0020_snapshot.json
@@ -0,0 +1,3661 @@
+{
+ "id": "86a7541b-16bf-4712-96e3-885ae9fc080e",
+ "prevId": "4df58052-f274-418d-a9ab-54e66bb81847",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.author_accounts": {
+ "name": "author_accounts",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "author_id": {
+ "name": "author_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "platform": {
+ "name": "platform",
+ "type": "author_platform",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "account_id": {
+ "name": "account_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "profile_url": {
+ "name": "profile_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_author_accounts_author_id": {
+ "name": "idx_author_accounts_author_id",
+ "columns": [
+ {
+ "expression": "author_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_author_accounts_platform_account_unique": {
+ "name": "idx_author_accounts_platform_account_unique",
+ "columns": [
+ {
+ "expression": "platform",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "account_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "author_accounts_author_id_authors_id_fk": {
+ "name": "author_accounts_author_id_authors_id_fk",
+ "tableFrom": "author_accounts",
+ "tableTo": "authors",
+ "columnsFrom": [
+ "author_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.authors": {
+ "name": "authors",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "account_id": {
+ "name": "account_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_authors_account_id": {
+ "name": "idx_authors_account_id",
+ "columns": [
+ {
+ "expression": "account_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_authors_name": {
+ "name": "idx_authors_name",
+ "columns": [
+ {
+ "expression": "name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.categories": {
+ "name": "categories",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "color": {
+ "name": "color",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'#808080'"
+ },
+ "parent_id": {
+ "name": "parent_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "categories_parent_id_categories_id_fk": {
+ "name": "categories_parent_id_categories_id_fk",
+ "tableFrom": "categories",
+ "tableTo": "categories",
+ "columnsFrom": [
+ "parent_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "categories_name_unique": {
+ "name": "categories_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ccip_embeddings": {
+ "name": "ccip_embeddings",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "region_id": {
+ "name": "region_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "embedding": {
+ "name": "embedding",
+ "type": "vector(768)",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "model": {
+ "name": "model",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "embedding_version": {
+ "name": "embedding_version",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "media_modified_at": {
+ "name": "media_modified_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "input_revision": {
+ "name": "input_revision",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "preprocessing_profile": {
+ "name": "preprocessing_profile",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'dghs-imgutils-rs/full-image-default/v1'"
+ },
+ "extracted_at": {
+ "name": "extracted_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_ccip_embeddings_region_id": {
+ "name": "idx_ccip_embeddings_region_id",
+ "columns": [
+ {
+ "expression": "region_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "ccip_embeddings_region_id_media_regions_id_fk": {
+ "name": "ccip_embeddings_region_id_media_regions_id_fk",
+ "tableFrom": "ccip_embeddings",
+ "tableTo": "media_regions",
+ "columnsFrom": [
+ "region_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "uq_ccip_embeddings_region_model_version": {
+ "name": "uq_ccip_embeddings_region_model_version",
+ "nullsNotDistinct": false,
+ "columns": [
+ "region_id",
+ "model",
+ "embedding_version",
+ "preprocessing_profile"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.character_ips": {
+ "name": "character_ips",
+ "schema": "",
+ "columns": {
+ "character_id": {
+ "name": "character_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "ip_id": {
+ "name": "ip_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ }
+ },
+ "indexes": {
+ "idx_character_ips_ip_id_character_id": {
+ "name": "idx_character_ips_ip_id_character_id",
+ "columns": [
+ {
+ "expression": "ip_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "character_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "character_ips_character_id_characters_id_fk": {
+ "name": "character_ips_character_id_characters_id_fk",
+ "tableFrom": "character_ips",
+ "tableTo": "characters",
+ "columnsFrom": [
+ "character_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "character_ips_ip_id_ips_id_fk": {
+ "name": "character_ips_ip_id_ips_id_fk",
+ "tableFrom": "character_ips",
+ "tableTo": "ips",
+ "columnsFrom": [
+ "ip_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "character_ips_character_id_ip_id_pk": {
+ "name": "character_ips_character_id_ip_id_pk",
+ "columns": [
+ "character_id",
+ "ip_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.characters": {
+ "name": "characters",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ },
+ "aliases": {
+ "name": "aliases",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "characters_name_unique": {
+ "name": "characters_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.collections": {
+ "name": "collections",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "collections_user_id_users_id_fk": {
+ "name": "collections_user_id_users_id_fk",
+ "tableFrom": "collections",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ips": {
+ "name": "ips",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "ips_name_unique": {
+ "name": "ips_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.jobs": {
+ "name": "jobs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "type": {
+ "name": "type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_id": {
+ "name": "source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "job_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "queue_name": {
+ "name": "queue_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "target_id": {
+ "name": "target_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "input_revision": {
+ "name": "input_revision",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "dedupe_key": {
+ "name": "dedupe_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "concurrency_key": {
+ "name": "concurrency_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "available_at": {
+ "name": "available_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "attempt_count": {
+ "name": "attempt_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "max_attempts": {
+ "name": "max_attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 5
+ },
+ "lease_duration_ms": {
+ "name": "lease_duration_ms",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 300000
+ },
+ "claim_token": {
+ "name": "claim_token",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "claimed_by": {
+ "name": "claimed_by",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "claimed_at": {
+ "name": "claimed_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "heartbeat_at": {
+ "name": "heartbeat_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "error_code": {
+ "name": "error_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "payload": {
+ "name": "payload",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "result": {
+ "name": "result",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "error": {
+ "name": "error",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "parent_id": {
+ "name": "parent_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "idx_jobs_pending_created": {
+ "name": "idx_jobs_pending_created",
+ "columns": [
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "where": "\"jobs\".\"status\" = 'pending' AND \"jobs\".\"type\" <> 'import_request'",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_jobs_pending_type_created": {
+ "name": "idx_jobs_pending_type_created",
+ "columns": [
+ {
+ "expression": "type",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "where": "\"jobs\".\"status\" = 'pending' AND \"jobs\".\"type\" <> 'import_request'",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_jobs_pending_lancedb_source": {
+ "name": "idx_jobs_pending_lancedb_source",
+ "columns": [
+ {
+ "expression": "source_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "where": "\"jobs\".\"status\" = 'pending'\n\t\t\t\t\tAND \"jobs\".\"type\" IN ('sync_lancedb', 'sync_lancedb_full', 'sync_lancedb_delta')\n\t\t\t\t\tAND \"jobs\".\"source_id\" IS NOT NULL",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_jobs_active_lancedb_source": {
+ "name": "idx_jobs_active_lancedb_source",
+ "columns": [
+ {
+ "expression": "source_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "where": "\"jobs\".\"status\" = 'in_progress'\n\t\t\t\t\tAND \"jobs\".\"type\" IN ('sync_lancedb', 'sync_lancedb_full', 'sync_lancedb_delta')\n\t\t\t\t\tAND \"jobs\".\"source_id\" IS NOT NULL",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_jobs_claim": {
+ "name": "idx_jobs_claim",
+ "columns": [
+ {
+ "expression": "queue_name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "available_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "where": "\"jobs\".\"status\" = 'pending'",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_jobs_stale_lease": {
+ "name": "idx_jobs_stale_lease",
+ "columns": [
+ {
+ "expression": "heartbeat_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "claimed_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "where": "\"jobs\".\"status\" = 'in_progress'",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_jobs_parent_status": {
+ "name": "idx_jobs_parent_status",
+ "columns": [
+ {
+ "expression": "parent_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "status",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_jobs_status_updated": {
+ "name": "idx_jobs_status_updated",
+ "columns": [
+ {
+ "expression": "status",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "updated_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "uq_jobs_active_dedupe": {
+ "name": "uq_jobs_active_dedupe",
+ "columns": [
+ {
+ "expression": "dedupe_key",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"jobs\".\"dedupe_key\" IS NOT NULL AND \"jobs\".\"status\" IN ('pending', 'in_progress')",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "uq_jobs_running_concurrency": {
+ "name": "uq_jobs_running_concurrency",
+ "columns": [
+ {
+ "expression": "concurrency_key",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"jobs\".\"concurrency_key\" IS NOT NULL AND \"jobs\".\"status\" = 'in_progress'",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "jobs_source_id_media_sources_id_fk": {
+ "name": "jobs_source_id_media_sources_id_fk",
+ "tableFrom": "jobs",
+ "tableTo": "media_sources",
+ "columnsFrom": [
+ "source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "jobs_parent_id_jobs_id_fk": {
+ "name": "jobs_parent_id_jobs_id_fk",
+ "tableFrom": "jobs",
+ "tableTo": "jobs",
+ "columnsFrom": [
+ "parent_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {
+ "jobs_attempt_count_nonnegative": {
+ "name": "jobs_attempt_count_nonnegative",
+ "value": "\"jobs\".\"attempt_count\" >= 0"
+ },
+ "jobs_max_attempts_positive": {
+ "name": "jobs_max_attempts_positive",
+ "value": "\"jobs\".\"max_attempts\" > 0"
+ },
+ "jobs_lease_duration_positive": {
+ "name": "jobs_lease_duration_positive",
+ "value": "\"jobs\".\"lease_duration_ms\" > 0"
+ }
+ },
+ "isRLSEnabled": false
+ },
+ "public.lancedb_sync_dirty": {
+ "name": "lancedb_sync_dirty",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "source_id": {
+ "name": "source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "operation": {
+ "name": "operation",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'upsert'"
+ },
+ "generation": {
+ "name": "generation",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "attempts": {
+ "name": "attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "last_error": {
+ "name": "last_error",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_lancedb_sync_dirty_source_updated": {
+ "name": "idx_lancedb_sync_dirty_source_updated",
+ "columns": [
+ {
+ "expression": "source_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "updated_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "lancedb_sync_dirty_source_id_media_sources_id_fk": {
+ "name": "lancedb_sync_dirty_source_id_media_sources_id_fk",
+ "tableFrom": "lancedb_sync_dirty",
+ "tableTo": "media_sources",
+ "columnsFrom": [
+ "source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "lancedb_sync_dirty_source_media_unique": {
+ "name": "lancedb_sync_dirty_source_media_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "source_id",
+ "media_id"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_authors": {
+ "name": "media_authors",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "author_id": {
+ "name": "author_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "idx_media_authors_author_id_media_id": {
+ "name": "idx_media_authors_author_id_media_id",
+ "columns": [
+ {
+ "expression": "author_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_authors_media_id_media_id_fk": {
+ "name": "media_authors_media_id_media_id_fk",
+ "tableFrom": "media_authors",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_authors_author_id_authors_id_fk": {
+ "name": "media_authors_author_id_authors_id_fk",
+ "tableFrom": "media_authors",
+ "tableTo": "authors",
+ "columnsFrom": [
+ "author_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "media_authors_media_id_author_id_pk": {
+ "name": "media_authors_media_id_author_id_pk",
+ "columns": [
+ "media_id",
+ "author_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_categories": {
+ "name": "media_categories",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "category_id": {
+ "name": "category_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "idx_media_categories_category_id_media_id": {
+ "name": "idx_media_categories_category_id_media_id",
+ "columns": [
+ {
+ "expression": "category_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_categories_media_id_media_id_fk": {
+ "name": "media_categories_media_id_media_id_fk",
+ "tableFrom": "media_categories",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_categories_category_id_categories_id_fk": {
+ "name": "media_categories_category_id_categories_id_fk",
+ "tableFrom": "media_categories",
+ "tableTo": "categories",
+ "columnsFrom": [
+ "category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "media_categories_media_id_category_id_pk": {
+ "name": "media_categories_media_id_category_id_pk",
+ "columns": [
+ "media_id",
+ "category_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_characters": {
+ "name": "media_characters",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "character_id": {
+ "name": "character_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "confidence": {
+ "name": "confidence",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ }
+ },
+ "indexes": {
+ "idx_media_characters_character_id_media_id": {
+ "name": "idx_media_characters_character_id_media_id",
+ "columns": [
+ {
+ "expression": "character_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_characters_media_id_media_id_fk": {
+ "name": "media_characters_media_id_media_id_fk",
+ "tableFrom": "media_characters",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_characters_character_id_characters_id_fk": {
+ "name": "media_characters_character_id_characters_id_fk",
+ "tableFrom": "media_characters",
+ "tableTo": "characters",
+ "columnsFrom": [
+ "character_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "media_characters_media_id_character_id_pk": {
+ "name": "media_characters_media_id_character_id_pk",
+ "columns": [
+ "media_id",
+ "character_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_collections": {
+ "name": "media_collections",
+ "schema": "",
+ "columns": {
+ "collection_id": {
+ "name": "collection_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "display_order": {
+ "name": "display_order",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "idx_media_collections_media_id": {
+ "name": "idx_media_collections_media_id",
+ "columns": [
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_collections_collection_id_collections_id_fk": {
+ "name": "media_collections_collection_id_collections_id_fk",
+ "tableFrom": "media_collections",
+ "tableTo": "collections",
+ "columnsFrom": [
+ "collection_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_collections_media_id_media_id_fk": {
+ "name": "media_collections_media_id_media_id_fk",
+ "tableFrom": "media_collections",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "media_collections_collection_id_media_id_pk": {
+ "name": "media_collections_collection_id_media_id_pk",
+ "columns": [
+ "collection_id",
+ "media_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_details": {
+ "name": "media_details",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "rating": {
+ "name": "rating",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "default": 0
+ },
+ "favorite": {
+ "name": "favorite",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false,
+ "default": false
+ },
+ "view_count": {
+ "name": "view_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "default": 0
+ },
+ "last_viewed_at": {
+ "name": "last_viewed_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'1970-01-01 00:00:00'"
+ }
+ },
+ "indexes": {
+ "idx_media_details_rating": {
+ "name": "idx_media_details_rating",
+ "columns": [
+ {
+ "expression": "rating",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_details_favorite": {
+ "name": "idx_media_details_favorite",
+ "columns": [
+ {
+ "expression": "favorite",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_details_view_count": {
+ "name": "idx_media_details_view_count",
+ "columns": [
+ {
+ "expression": "view_count",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_details_media_id_media_id_fk": {
+ "name": "media_details_media_id_media_id_fk",
+ "tableFrom": "media_details",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_generation_info": {
+ "name": "media_generation_info",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "metadata": {
+ "name": "metadata",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "prompt": {
+ "name": "prompt",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "negative_prompt": {
+ "name": "negative_prompt",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "workflow": {
+ "name": "workflow",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "loras": {
+ "name": "loras",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "vae": {
+ "name": "vae",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "hypernetworks": {
+ "name": "hypernetworks",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "embeddings": {
+ "name": "embeddings",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "ai_generated": {
+ "name": "ai_generated",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false,
+ "default": false
+ },
+ "model_name": {
+ "name": "model_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "seed": {
+ "name": "seed",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": false,
+ "default": -1
+ },
+ "cfg_scale": {
+ "name": "cfg_scale",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "default": 0
+ },
+ "steps": {
+ "name": "steps",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "default": 0
+ }
+ },
+ "indexes": {
+ "idx_media_generation_info_metadata": {
+ "name": "idx_media_generation_info_metadata",
+ "columns": [
+ {
+ "expression": "metadata",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_generation_info_ai_generated": {
+ "name": "idx_media_generation_info_ai_generated",
+ "columns": [
+ {
+ "expression": "ai_generated",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_generation_info_model_name": {
+ "name": "idx_media_generation_info_model_name",
+ "columns": [
+ {
+ "expression": "model_name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_generation_info_media_id_media_id_fk": {
+ "name": "media_generation_info_media_id_media_id_fk",
+ "tableFrom": "media_generation_info",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_ips": {
+ "name": "media_ips",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "ip_id": {
+ "name": "ip_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "confidence": {
+ "name": "confidence",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ }
+ },
+ "indexes": {
+ "idx_media_ips_ip_id_media_id": {
+ "name": "idx_media_ips_ip_id_media_id",
+ "columns": [
+ {
+ "expression": "ip_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_ips_media_id_media_id_fk": {
+ "name": "media_ips_media_id_media_id_fk",
+ "tableFrom": "media_ips",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_ips_ip_id_ips_id_fk": {
+ "name": "media_ips_ip_id_ips_id_fk",
+ "tableFrom": "media_ips",
+ "tableTo": "ips",
+ "columnsFrom": [
+ "ip_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "media_ips_media_id_ip_id_pk": {
+ "name": "media_ips_media_id_ip_id_pk",
+ "columns": [
+ "media_id",
+ "ip_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_projects": {
+ "name": "media_projects",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "idx_media_projects_project_id_media_id": {
+ "name": "idx_media_projects_project_id_media_id",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_projects_media_id_media_id_fk": {
+ "name": "media_projects_media_id_media_id_fk",
+ "tableFrom": "media_projects",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_projects_project_id_projects_id_fk": {
+ "name": "media_projects_project_id_projects_id_fk",
+ "tableFrom": "media_projects",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "media_projects_media_id_project_id_pk": {
+ "name": "media_projects_media_id_project_id_pk",
+ "columns": [
+ "media_id",
+ "project_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_regions": {
+ "name": "media_regions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "kind": {
+ "name": "kind",
+ "type": "media_region_kind",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "x": {
+ "name": "x",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "y": {
+ "name": "y",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "width": {
+ "name": "width",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "height": {
+ "name": "height",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_modified_at": {
+ "name": "source_modified_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_width": {
+ "name": "source_width",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_height": {
+ "name": "source_height",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_revision": {
+ "name": "source_revision",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "region_revision": {
+ "name": "region_revision",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "label": {
+ "name": "label",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "manual_reason": {
+ "name": "manual_reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "detection_key": {
+ "name": "detection_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "detector": {
+ "name": "detector",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "detector_model": {
+ "name": "detector_model",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "detector_version": {
+ "name": "detector_version",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "score": {
+ "name": "score",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_media_regions_media_id": {
+ "name": "idx_media_regions_media_id",
+ "columns": [
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "uq_media_regions_full_media_id": {
+ "name": "uq_media_regions_full_media_id",
+ "columns": [
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"media_regions\".\"kind\" = 'full'",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "uq_media_regions_detection_key": {
+ "name": "uq_media_regions_detection_key",
+ "columns": [
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "detection_key",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"media_regions\".\"detection_key\" IS NOT NULL",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_regions_media_id_media_id_fk": {
+ "name": "media_regions_media_id_media_id_fk",
+ "tableFrom": "media_regions",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {
+ "media_regions_bbox_by_kind": {
+ "name": "media_regions_bbox_by_kind",
+ "value": "(\n\t\t\t\t(\"media_regions\".\"kind\" = 'full' AND \"media_regions\".\"x\" IS NULL AND \"media_regions\".\"y\" IS NULL AND \"media_regions\".\"width\" IS NULL AND \"media_regions\".\"height\" IS NULL)\n\t\t\t\tOR\n\t\t\t\t(\"media_regions\".\"kind\" <> 'full' AND \"media_regions\".\"x\" IS NOT NULL AND \"media_regions\".\"y\" IS NOT NULL AND \"media_regions\".\"width\" IS NOT NULL AND \"media_regions\".\"height\" IS NOT NULL\n\t\t\t\t\tAND \"media_regions\".\"x\" >= 0 AND \"media_regions\".\"y\" >= 0 AND \"media_regions\".\"width\" > 0 AND \"media_regions\".\"height\" > 0\n\t\t\t\t\tAND \"media_regions\".\"x\" + \"media_regions\".\"width\" <= 1 AND \"media_regions\".\"y\" + \"media_regions\".\"height\" <= 1)\n\t\t\t)"
+ },
+ "media_regions_score_range": {
+ "name": "media_regions_score_range",
+ "value": "\"media_regions\".\"score\" IS NULL OR (\"media_regions\".\"score\" >= 0 AND \"media_regions\".\"score\" <= 1)"
+ },
+ "media_regions_source_dimensions_positive": {
+ "name": "media_regions_source_dimensions_positive",
+ "value": "\"media_regions\".\"source_width\" > 0 AND \"media_regions\".\"source_height\" > 0"
+ }
+ },
+ "isRLSEnabled": false
+ },
+ "public.media_relations": {
+ "name": "media_relations",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "parent_media_id": {
+ "name": "parent_media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "child_media_id": {
+ "name": "child_media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "relation_type": {
+ "name": "relation_type",
+ "type": "media_relation_type",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "order_index": {
+ "name": "order_index",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "metadata": {
+ "name": "metadata",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_region_id": {
+ "name": "source_region_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "derivation_key": {
+ "name": "derivation_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_media_relations_child": {
+ "name": "idx_media_relations_child",
+ "columns": [
+ {
+ "expression": "child_media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_relations_type": {
+ "name": "idx_media_relations_type",
+ "columns": [
+ {
+ "expression": "relation_type",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_relations_source_region": {
+ "name": "idx_media_relations_source_region",
+ "columns": [
+ {
+ "expression": "source_region_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "uq_media_relations_derivation_key": {
+ "name": "uq_media_relations_derivation_key",
+ "columns": [
+ {
+ "expression": "derivation_key",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"media_relations\".\"derivation_key\" IS NOT NULL",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_relations_parent_media_id_media_id_fk": {
+ "name": "media_relations_parent_media_id_media_id_fk",
+ "tableFrom": "media_relations",
+ "tableTo": "media",
+ "columnsFrom": [
+ "parent_media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_relations_child_media_id_media_id_fk": {
+ "name": "media_relations_child_media_id_media_id_fk",
+ "tableFrom": "media_relations",
+ "tableTo": "media",
+ "columnsFrom": [
+ "child_media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_relations_source_region_id_media_regions_id_fk": {
+ "name": "media_relations_source_region_id_media_regions_id_fk",
+ "tableFrom": "media_relations",
+ "tableTo": "media_regions",
+ "columnsFrom": [
+ "source_region_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "parent_child_type_unique": {
+ "name": "parent_child_type_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "parent_media_id",
+ "child_media_id",
+ "relation_type"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_sources": {
+ "name": "media_sources",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "type": {
+ "name": "type",
+ "type": "media_source_type",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "connection_info": {
+ "name": "connection_info",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_sync": {
+ "name": "media_sync",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "sync_status": {
+ "name": "sync_status",
+ "type": "media_sync_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'synced'"
+ },
+ "backup_urls": {
+ "name": "backup_urls",
+ "type": "text[]",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'{}'"
+ },
+ "last_synced_at": {
+ "name": "last_synced_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sync_attempts": {
+ "name": "sync_attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "default": 0
+ },
+ "last_error": {
+ "name": "last_error",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "media_sync_media_id_media_id_fk": {
+ "name": "media_sync_media_id_media_id_fk",
+ "tableFrom": "media_sync",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_tags": {
+ "name": "media_tags",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "tag_id": {
+ "name": "tag_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "tag_type": {
+ "name": "tag_type",
+ "type": "tag_type",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'positive'"
+ },
+ "confidence": {
+ "name": "confidence",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ }
+ },
+ "indexes": {
+ "idx_media_tags_tag_id_tag_type_media_id": {
+ "name": "idx_media_tags_tag_id_tag_type_media_id",
+ "columns": [
+ {
+ "expression": "tag_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "tag_type",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_tags_media_id_media_id_fk": {
+ "name": "media_tags_media_id_media_id_fk",
+ "tableFrom": "media_tags",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "media_tags_tag_id_tags_id_fk": {
+ "name": "media_tags_tag_id_tags_id_fk",
+ "tableFrom": "media_tags",
+ "tableTo": "tags",
+ "columnsFrom": [
+ "tag_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "media_tags_media_id_tag_id_tag_type_pk": {
+ "name": "media_tags_media_id_tag_id_tag_type_pk",
+ "columns": [
+ "media_id",
+ "tag_id",
+ "tag_type"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_technical_info": {
+ "name": "media_technical_info",
+ "schema": "",
+ "columns": {
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "color_profile": {
+ "name": "color_profile",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "exif_data": {
+ "name": "exif_data",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'{}'"
+ },
+ "hash_md5": {
+ "name": "hash_md5",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "hash_perceptual": {
+ "name": "hash_perceptual",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "duration_seconds": {
+ "name": "duration_seconds",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "frame_rate": {
+ "name": "frame_rate",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "bitrate_kbps": {
+ "name": "bitrate_kbps",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "video_codec": {
+ "name": "video_codec",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "audio_codec": {
+ "name": "audio_codec",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "idx_media_technical_info_hash_md5": {
+ "name": "idx_media_technical_info_hash_md5",
+ "columns": [
+ {
+ "expression": "hash_md5",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_technical_info_media_id_media_id_fk": {
+ "name": "media_technical_info_media_id_media_id_fk",
+ "tableFrom": "media_technical_info",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media_urls": {
+ "name": "media_urls",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_media_urls_media_id": {
+ "name": "idx_media_urls_media_id",
+ "columns": [
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_urls_url": {
+ "name": "idx_media_urls_url",
+ "columns": [
+ {
+ "expression": "url",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_urls_media_id_url_unique": {
+ "name": "idx_media_urls_media_id_url_unique",
+ "columns": [
+ {
+ "expression": "media_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "url",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_urls_media_id_media_id_fk": {
+ "name": "media_urls_media_id_media_id_fk",
+ "tableFrom": "media_urls",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.media": {
+ "name": "media",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "source_id": {
+ "name": "source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_path": {
+ "name": "file_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_name": {
+ "name": "file_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "media_type": {
+ "name": "media_type",
+ "type": "media_type",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "width": {
+ "name": "width",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "height": {
+ "name": "height",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_size": {
+ "name": "file_size",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "modified_at": {
+ "name": "modified_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "indexed_at": {
+ "name": "indexed_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "status": {
+ "name": "status",
+ "type": "media_organization_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'active'"
+ }
+ },
+ "indexes": {
+ "idx_media_source_id": {
+ "name": "idx_media_source_id",
+ "columns": [
+ {
+ "expression": "source_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_file_size": {
+ "name": "idx_media_file_size",
+ "columns": [
+ {
+ "expression": "file_size",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_file_name": {
+ "name": "idx_media_file_name",
+ "columns": [
+ {
+ "expression": "file_name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_created_at": {
+ "name": "idx_media_created_at",
+ "columns": [
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "idx_media_description": {
+ "name": "idx_media_description",
+ "columns": [
+ {
+ "expression": "description",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "media_source_id_media_sources_id_fk": {
+ "name": "media_source_id_media_sources_id_fk",
+ "tableFrom": "media",
+ "tableTo": "media_sources",
+ "columnsFrom": [
+ "source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "source_id_file_path_unique": {
+ "name": "source_id_file_path_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "source_id",
+ "file_path"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.presets": {
+ "name": "presets",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "value": {
+ "name": "value",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "sort": {
+ "name": "sort",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "order": {
+ "name": "order",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "mode": {
+ "name": "mode",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "presets_name_unique": {
+ "name": "presets_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.projects": {
+ "name": "projects",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "idx_projects_name": {
+ "name": "idx_projects_name",
+ "columns": [
+ {
+ "expression": "name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "projects_name_unique": {
+ "name": "projects_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.similar_media": {
+ "name": "similar_media",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "media1_id": {
+ "name": "media1_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "media2_id": {
+ "name": "media2_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "similarity_score": {
+ "name": "similarity_score",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "default": 0
+ },
+ "algorithm": {
+ "name": "algorithm",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'perceptual'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_similar_media_score": {
+ "name": "idx_similar_media_score",
+ "columns": [
+ {
+ "expression": "similarity_score",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "similar_media_media1_id_media_id_fk": {
+ "name": "similar_media_media1_id_media_id_fk",
+ "tableFrom": "similar_media",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media1_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "similar_media_media2_id_media_id_fk": {
+ "name": "similar_media_media2_id_media_id_fk",
+ "tableFrom": "similar_media",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media2_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "media1Id_media2Id_algorithm_unique": {
+ "name": "media1Id_media2Id_algorithm_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "media1_id",
+ "media2_id",
+ "algorithm"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.tags": {
+ "name": "tags",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "attribute": {
+ "name": "attribute",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "color": {
+ "name": "color",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'manual'"
+ },
+ "author_id": {
+ "name": "author_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "idx_tags_author_id": {
+ "name": "idx_tags_author_id",
+ "columns": [
+ {
+ "expression": "author_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "tags_author_id_authors_id_fk": {
+ "name": "tags_author_id_authors_id_fk",
+ "tableFrom": "tags",
+ "tableTo": "authors",
+ "columnsFrom": [
+ "author_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "tags_name_unique": {
+ "name": "tags_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.users": {
+ "name": "users",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.view_history": {
+ "name": "view_history",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "media_id": {
+ "name": "media_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "viewed_at": {
+ "name": "viewed_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "ip_address": {
+ "name": "ip_address",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "user_agent": {
+ "name": "user_agent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "''"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "view_history_media_id_media_id_fk": {
+ "name": "view_history_media_id_media_id_fk",
+ "tableFrom": "view_history",
+ "tableTo": "media",
+ "columnsFrom": [
+ "media_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.author_platform": {
+ "name": "author_platform",
+ "schema": "public",
+ "values": [
+ "twitter",
+ "pixiv-fanbox",
+ "danbooru"
+ ]
+ },
+ "public.job_status": {
+ "name": "job_status",
+ "schema": "public",
+ "values": [
+ "pending",
+ "in_progress",
+ "completed",
+ "failed",
+ "cancelled"
+ ]
+ },
+ "public.media_organization_status": {
+ "name": "media_organization_status",
+ "schema": "public",
+ "values": [
+ "active",
+ "archived",
+ "deleted"
+ ]
+ },
+ "public.media_region_kind": {
+ "name": "media_region_kind",
+ "schema": "public",
+ "values": [
+ "full",
+ "person",
+ "manual"
+ ]
+ },
+ "public.media_relation_type": {
+ "name": "media_relation_type",
+ "schema": "public",
+ "values": [
+ "variant",
+ "version",
+ "page",
+ "derivative",
+ "edit",
+ "source"
+ ]
+ },
+ "public.media_source_type": {
+ "name": "media_source_type",
+ "schema": "public",
+ "values": [
+ "local",
+ "sftp",
+ "s3"
+ ]
+ },
+ "public.media_sync_status": {
+ "name": "media_sync_status",
+ "schema": "public",
+ "values": [
+ "synced",
+ "pending",
+ "failed"
+ ]
+ },
+ "public.media_type": {
+ "name": "media_type",
+ "schema": "public",
+ "values": [
+ "image",
+ "video",
+ "audio"
+ ]
+ },
+ "public.tag_type": {
+ "name": "tag_type",
+ "schema": "public",
+ "values": [
+ "positive",
+ "negative"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
diff --git a/apps/server/drizzle/meta/_journal.json b/apps/server/drizzle/meta/_journal.json
index 7a100fc3d..407def906 100644
--- a/apps/server/drizzle/meta/_journal.json
+++ b/apps/server/drizzle/meta/_journal.json
@@ -141,6 +141,13 @@
"when": 1784389747874,
"tag": "0019_purple_devos",
"breakpoints": true
+ },
+ {
+ "idx": 20,
+ "version": "7",
+ "when": 1784815653388,
+ "tag": "0020_bizarre_otto_octavius",
+ "breakpoints": true
}
]
-}
+}
\ No newline at end of file
diff --git a/apps/server/nitro.config.ts b/apps/server/nitro.config.ts
index e32fe49c4..16f51c3f7 100644
--- a/apps/server/nitro.config.ts
+++ b/apps/server/nitro.config.ts
@@ -28,24 +28,54 @@ export default defineNitroConfig({
fs.existsSync(pgliteLocalPath) ? pgliteLocalPath : pgliteRootPath,
);
const pgliteDistPath = path.join(pglitePkgPath, "dist");
+ const pgvectorLocalPath = path.resolve(
+ __dirname,
+ "node_modules/@electric-sql/pglite-pgvector/package.json",
+ );
+ const pgvectorRootPath = path.resolve(
+ __dirname,
+ "../../node_modules/@electric-sql/pglite-pgvector/package.json",
+ );
+ const pgvectorPkgPath = path.dirname(
+ fs.existsSync(pgvectorLocalPath) ? pgvectorLocalPath : pgvectorRootPath,
+ );
- const assetsToCopy = ["pglite.data", "pglite.wasm"];
+ const assetsToCopy = [
+ {
+ source: path.join(pgliteDistPath, "pglite.data"),
+ destination: path.join(libsDir, "pglite.data"),
+ },
+ {
+ source: path.join(pgliteDistPath, "pglite.wasm"),
+ destination: path.join(libsDir, "pglite.wasm"),
+ },
+ {
+ source: path.join(pgvectorPkgPath, "dist", "vector.tar.gz"),
+ destination: path.join(libsDir, "vector.tar.gz"),
+ },
+ ];
- for (const asset of assetsToCopy) {
- const source = path.join(pgliteDistPath, asset);
- const destination = path.join(libsDir, asset);
-
- if (fs.existsSync(source)) {
- if (!fs.existsSync(libsDir)) {
- fs.mkdirSync(libsDir, { recursive: true });
- }
- fs.copyFileSync(source, destination);
- console.log(`[Nitro] Successfully copied ${asset} to ${destination}`);
- } else {
- console.warn(`[Nitro] Warning: ${asset} not found at ${source}`);
- }
+ fs.mkdirSync(libsDir, { recursive: true });
+ for (const asset of assetsToCopy) {
+ if (!fs.existsSync(asset.source) || fs.statSync(asset.source).size === 0) {
+ throw new Error(
+ `Required PGlite runtime asset is missing or empty: ${asset.source}`,
+ );
+ }
+ fs.copyFileSync(asset.source, asset.destination);
+ console.log(
+ `[Nitro] Successfully copied ${path.basename(asset.source)} to ${asset.destination}`,
+ );
}
+ const migrationsSource = path.join(__dirname, "drizzle");
+ const migrationsDestination = path.join(serverDir, "drizzle");
+ const journalSource = path.join(migrationsSource, "meta", "_journal.json");
+ if (!fs.existsSync(journalSource) || fs.statSync(journalSource).size === 0) {
+ throw new Error(`Drizzle migration journal is missing or empty: ${journalSource}`);
+ }
+ fs.cpSync(migrationsSource, migrationsDestination, { recursive: true });
+
// Copy yt-dlp binary for bundled youtube-dl-exec
const ytDlpLocalPath = path.resolve(__dirname, "node_modules/youtube-dl-exec/bin/yt-dlp");
const ytDlpRootPath = path.resolve(
diff --git a/apps/server/package.json b/apps/server/package.json
index 592308bfe..69466f132 100644
--- a/apps/server/package.json
+++ b/apps/server/package.json
@@ -17,6 +17,8 @@
"db:drop": "drizzle-kit drop",
"db:dump": "bun scripts/dump-db.ts",
"db:restore": "bun scripts/restore-db.ts",
+ "db:validate-rehearsal": "bun scripts/validate-postgres-rehearsal.ts",
+ "db:set-jobs-logged": "bun scripts/set-jobs-logged.ts",
"ccip:migrate-from-lancedb": "bun scripts/migrate-ccip-lancedb.ts",
"lancedb:sync-slow": "bun scripts/sync-lancedb-slow.ts",
"measure:dev-startup": "bun scripts/measure-dev-startup.ts",
@@ -27,6 +29,7 @@
"test": "bun run test:unit && bun run test:integration && bun run test:e2e",
"test:unit": "vp test run -c vitest.unit.config.ts",
"test:integration": "vp test run -c vitest.integration.config.ts",
+ "test:pglite-bundle": "bun scripts/verify-pglite-bundle.ts",
"test:e2e": "bun scripts/run-e2e.ts --mode=all",
"test:e2e:dev": "bun scripts/run-e2e.ts --mode=dev",
"test:e2e:production": "bun scripts/run-e2e.ts --mode=production",
diff --git a/apps/server/public/openapi.json b/apps/server/public/openapi.json
index f5a11829b..f182dced1 100644
--- a/apps/server/public/openapi.json
+++ b/apps/server/public/openapi.json
@@ -67,7 +67,9 @@
"operationId": "sources.list",
"summary": "メディアソース一覧取得",
"description": "登録されているすべてのメディアソース（ローカル、SFTP、S3等）を取得します。",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -92,7 +94,9 @@
"operationId": "sources.get",
"summary": "メディアソース詳細取得",
"description": "UUIDを指定して特定のメディアソースの情報を取得します。",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -117,7 +121,9 @@
"operationId": "sources.create",
"summary": "メディアソース作成",
"description": "新しいメディアソースを登録します。",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -142,7 +148,9 @@
"operationId": "sources.update",
"summary": "メディアソース更新",
"description": "既存のメディアソースの設定を更新します。",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -167,7 +175,9 @@
"operationId": "sources.delete",
"summary": "メディアソース削除",
"description": "メディアソースを削除し、監視を停止します。",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -191,7 +201,9 @@
"post": {
"operationId": "sources.sync",
"summary": "sync",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -215,7 +227,9 @@
"post": {
"operationId": "sources.dump",
"summary": "dump",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -239,7 +253,9 @@
"post": {
"operationId": "sources.restore",
"summary": "restore",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -263,7 +279,9 @@
"post": {
"operationId": "sources.importZip",
"summary": "importZip",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -287,7 +305,9 @@
"post": {
"operationId": "sources.importNdjson",
"summary": "importNdjson",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -311,7 +331,9 @@
"post": {
"operationId": "sources.importLanceDB",
"summary": "importLanceDB",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -336,7 +358,9 @@
"operationId": "sources.status",
"summary": "メディアソースの状態取得",
"description": "スキャン進捗やファイル数などの統計情報を取得します。",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -360,7 +384,9 @@
"post": {
"operationId": "sources.events",
"summary": "events",
- "tags": ["Media Sources"],
+ "tags": [
+ "Media Sources"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -382,7 +408,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
},
{
"type": "object",
@@ -398,7 +426,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
},
{
"type": "object",
@@ -414,7 +444,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
}
]
}
@@ -428,7 +460,9 @@
"post": {
"operationId": "tags.list",
"summary": "list",
- "tags": ["Tags"],
+ "tags": [
+ "Tags"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -452,7 +486,9 @@
"post": {
"operationId": "tags.get",
"summary": "get",
- "tags": ["Tags"],
+ "tags": [
+ "Tags"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -476,7 +512,9 @@
"post": {
"operationId": "tags.create",
"summary": "create",
- "tags": ["Tags"],
+ "tags": [
+ "Tags"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -500,7 +538,9 @@
"post": {
"operationId": "tags.update",
"summary": "update",
- "tags": ["Tags"],
+ "tags": [
+ "Tags"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -524,7 +564,9 @@
"post": {
"operationId": "tags.delete",
"summary": "delete",
- "tags": ["Tags"],
+ "tags": [
+ "Tags"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -549,7 +591,9 @@
"operationId": "media.search",
"summary": "メディア検索",
"description": "タグ、プロジェクト、キャラクターなどの条件でメディアを検索します。",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -573,7 +617,9 @@
"post": {
"operationId": "media.searchSimilar",
"summary": "searchSimilar",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -597,7 +643,9 @@
"post": {
"operationId": "media.get",
"summary": "get",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -621,7 +669,9 @@
"post": {
"operationId": "media.getDetails",
"summary": "getDetails",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -645,7 +695,9 @@
"post": {
"operationId": "media.findDuplicates",
"summary": "findDuplicates",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -669,7 +721,9 @@
"post": {
"operationId": "media.getContent",
"summary": "getContent",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -693,7 +747,9 @@
"post": {
"operationId": "media.getTags",
"summary": "getTags",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -717,7 +773,9 @@
"post": {
"operationId": "media.update",
"summary": "update",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -741,7 +799,9 @@
"post": {
"operationId": "media.sync",
"summary": "sync",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -765,7 +825,9 @@
"post": {
"operationId": "media.delete",
"summary": "delete",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -789,7 +851,9 @@
"post": {
"operationId": "media.copy",
"summary": "copy",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -813,7 +877,9 @@
"post": {
"operationId": "media.move",
"summary": "move",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -837,7 +903,9 @@
"post": {
"operationId": "media.upload",
"summary": "upload",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -861,7 +929,9 @@
"post": {
"operationId": "media.bulkEdit",
"summary": "bulkEdit",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -885,7 +955,9 @@
"post": {
"operationId": "media.bulkDelete",
"summary": "bulkDelete",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -909,7 +981,9 @@
"post": {
"operationId": "media.bulkMove",
"summary": "bulkMove",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -933,7 +1007,9 @@
"post": {
"operationId": "media.bulkTag",
"summary": "bulkTag",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -957,7 +1033,9 @@
"post": {
"operationId": "media.bulkCopyToSource",
"summary": "bulkCopyToSource",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -981,7 +1059,124 @@
"post": {
"operationId": "media.bulkMoveToSource",
"summary": "bulkMoveToSource",
- "tags": ["Media"],
+ "tags": [
+ "Media"
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {},
+ {
+ "not": {}
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/mediaRegions/list": {
+ "post": {
+ "operationId": "mediaRegions.list",
+ "summary": "list",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {},
+ {
+ "not": {}
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/mediaRegions/createManual": {
+ "post": {
+ "operationId": "mediaRegions.createManual",
+ "summary": "createManual",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {},
+ {
+ "not": {}
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/mediaRegions/update": {
+ "post": {
+ "operationId": "mediaRegions.update",
+ "summary": "update",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {},
+ {
+ "not": {}
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/mediaRegions/delete": {
+ "post": {
+ "operationId": "mediaRegions.delete",
+ "summary": "delete",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {},
+ {
+ "not": {}
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/mediaRegions/materialize": {
+ "post": {
+ "operationId": "mediaRegions.materialize",
+ "summary": "materialize",
"responses": {
"200": {
"description": "OK",
@@ -1005,7 +1200,9 @@
"post": {
"operationId": "categories.list",
"summary": "list",
- "tags": ["Categories"],
+ "tags": [
+ "Categories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1029,7 +1226,9 @@
"post": {
"operationId": "categories.get",
"summary": "get",
- "tags": ["Categories"],
+ "tags": [
+ "Categories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1053,7 +1252,9 @@
"post": {
"operationId": "categories.create",
"summary": "create",
- "tags": ["Categories"],
+ "tags": [
+ "Categories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1077,7 +1278,9 @@
"post": {
"operationId": "categories.update",
"summary": "update",
- "tags": ["Categories"],
+ "tags": [
+ "Categories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1101,7 +1304,9 @@
"post": {
"operationId": "categories.delete",
"summary": "delete",
- "tags": ["Categories"],
+ "tags": [
+ "Categories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1125,7 +1330,9 @@
"post": {
"operationId": "projects.list",
"summary": "list",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1149,7 +1356,9 @@
"post": {
"operationId": "projects.get",
"summary": "get",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1173,7 +1382,9 @@
"post": {
"operationId": "projects.create",
"summary": "create",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1197,7 +1408,9 @@
"post": {
"operationId": "projects.update",
"summary": "update",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1221,7 +1434,9 @@
"post": {
"operationId": "projects.delete",
"summary": "delete",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1245,7 +1460,9 @@
"post": {
"operationId": "projects.listForMedia",
"summary": "listForMedia",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1269,7 +1486,9 @@
"post": {
"operationId": "projects.addToMedia",
"summary": "addToMedia",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1293,7 +1512,9 @@
"post": {
"operationId": "projects.removeFromMedia",
"summary": "removeFromMedia",
- "tags": ["Projects"],
+ "tags": [
+ "Projects"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1317,7 +1538,9 @@
"post": {
"operationId": "characters.list",
"summary": "list",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1341,7 +1564,9 @@
"post": {
"operationId": "characters.get",
"summary": "get",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1365,7 +1590,9 @@
"post": {
"operationId": "characters.create",
"summary": "create",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1389,7 +1616,9 @@
"post": {
"operationId": "characters.update",
"summary": "update",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1413,7 +1642,9 @@
"post": {
"operationId": "characters.delete",
"summary": "delete",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1437,7 +1668,9 @@
"post": {
"operationId": "characters.listForMedia",
"summary": "listForMedia",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1461,7 +1694,9 @@
"post": {
"operationId": "characters.addToMedia",
"summary": "addToMedia",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1485,7 +1720,9 @@
"post": {
"operationId": "characters.removeFromMedia",
"summary": "removeFromMedia",
- "tags": ["Characters"],
+ "tags": [
+ "Characters"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1509,7 +1746,9 @@
"post": {
"operationId": "ips.list",
"summary": "list",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1533,7 +1772,9 @@
"post": {
"operationId": "ips.get",
"summary": "get",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1557,7 +1798,9 @@
"post": {
"operationId": "ips.create",
"summary": "create",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1581,7 +1824,9 @@
"post": {
"operationId": "ips.update",
"summary": "update",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1605,7 +1850,9 @@
"post": {
"operationId": "ips.delete",
"summary": "delete",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1629,7 +1876,9 @@
"post": {
"operationId": "ips.listForMedia",
"summary": "listForMedia",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1653,7 +1902,9 @@
"post": {
"operationId": "ips.addToMedia",
"summary": "addToMedia",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1677,7 +1928,9 @@
"post": {
"operationId": "ips.removeFromMedia",
"summary": "removeFromMedia",
- "tags": ["IPs"],
+ "tags": [
+ "IPs"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1701,7 +1954,9 @@
"post": {
"operationId": "thumbnails.generate",
"summary": "generate",
- "tags": ["Thumbnails"],
+ "tags": [
+ "Thumbnails"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1725,7 +1980,9 @@
"post": {
"operationId": "thumbnails.clear",
"summary": "clear",
- "tags": ["Thumbnails"],
+ "tags": [
+ "Thumbnails"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1749,7 +2006,9 @@
"post": {
"operationId": "downloads.start",
"summary": "start",
- "tags": ["Downloads"],
+ "tags": [
+ "Downloads"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1773,7 +2032,9 @@
"post": {
"operationId": "directories.list",
"summary": "list",
- "tags": ["Directories"],
+ "tags": [
+ "Directories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1797,7 +2058,9 @@
"post": {
"operationId": "directories.create",
"summary": "create",
- "tags": ["Directories"],
+ "tags": [
+ "Directories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1821,7 +2084,9 @@
"post": {
"operationId": "directories.delete",
"summary": "delete",
- "tags": ["Directories"],
+ "tags": [
+ "Directories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1845,7 +2110,9 @@
"post": {
"operationId": "directories.rename",
"summary": "rename",
- "tags": ["Directories"],
+ "tags": [
+ "Directories"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1870,7 +2137,35 @@
"operationId": "ai.tag",
"summary": "AI自動タグ付け",
"description": "画像を解析して、関連するタグ（DeepDanbooru等）を自動生成します。",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {},
+ {
+ "not": {}
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/ai/tagOppaiOracle": {
+ "post": {
+ "operationId": "ai.tagOppaiOracle",
+ "summary": "tagOppaiOracle",
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1894,7 +2189,9 @@
"post": {
"operationId": "ai.ccipFeature",
"summary": "ccipFeature",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1918,7 +2215,9 @@
"post": {
"operationId": "ai.ccipDifference",
"summary": "ccipDifference",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1942,7 +2241,9 @@
"post": {
"operationId": "ai.ccipDistances",
"summary": "ccipDistances",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1966,7 +2267,9 @@
"post": {
"operationId": "ai.scanBatchTaggingTargets",
"summary": "scanBatchTaggingTargets",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -1990,7 +2293,9 @@
"post": {
"operationId": "ai.startBatchTagging",
"summary": "startBatchTagging",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -2014,7 +2319,9 @@
"post": {
"operationId": "ai.ccipVectorStatus",
"summary": "ccipVectorStatus",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -2038,7 +2345,9 @@
"post": {
"operationId": "ai.startCcipExtraction",
"summary": "startCcipExtraction",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -2062,7 +2371,9 @@
"post": {
"operationId": "ai.scanBatchCcipTargets",
"summary": "scanBatchCcipTargets",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -2086,7 +2397,9 @@
"post": {
"operationId": "ai.startBatchCcipExtraction",
"summary": "startBatchCcipExtraction",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -2110,7 +2423,9 @@
"post": {
"operationId": "ai.detectAndCropCharacters",
"summary": "detectAndCropCharacters",
- "tags": ["AI"],
+ "tags": [
+ "AI"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -2157,7 +2472,9 @@
"post": {
"operationId": "utils.fetchUrl",
"summary": "fetchUrl",
- "tags": ["Utilities"],
+ "tags": [
+ "Utilities"
+ ],
"responses": {
"200": {
"description": "OK",
@@ -2294,7 +2611,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
},
{
"type": "object",
@@ -2310,7 +2629,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
},
{
"type": "object",
@@ -2326,7 +2647,32 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/jobs/get": {
+ "post": {
+ "operationId": "jobs.get",
+ "summary": "get",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {},
+ {
+ "not": {}
}
]
}
@@ -2361,7 +2707,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
},
{
"type": "object",
@@ -2377,7 +2725,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
},
{
"type": "object",
@@ -2393,7 +2743,9 @@
"type": "number"
}
},
- "required": ["event"]
+ "required": [
+ "event"
+ ]
}
]
}
@@ -2588,4 +2940,4 @@
}
}
}
-}
+}
\ No newline at end of file
diff --git a/apps/server/scripts/dump-db.ts b/apps/server/scripts/dump-db.ts
index 18dca4735..23d1dd120 100644
--- a/apps/server/scripts/dump-db.ts
+++ b/apps/server/scripts/dump-db.ts
@@ -1,49 +1,121 @@
///
-import { $ } from "bun";
-import { mkdir } from "node:fs/promises";
+import { access, mkdir, rename, rm, stat } from "node:fs/promises";
import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { logger } from "../src/infrastructure/logger";
-// Load environment variables
-const DB_USER = process.env.DB_USER || "postgres";
-const DB_DATABASE = process.env.DB_DATABASE || "solid-imager";
-const CONTAINER_NAME = "solid-imager-db-1"; // Assuming default naming convention, or retrieve from docker-compose
-
-// Backup configuration
-const BACKUP_DIR = "backups";
-const TIMESTAMP = new Date().toISOString().replace(/[:.]/g, "-");
-const FILENAME = `backup-${TIMESTAMP}.sql`;
-const FILEPATH = path.join(BACKUP_DIR, FILENAME);
-
-console.log("📦 Starting database backup...");
-
-try {
- // Ensure backup directory exists
- await mkdir(BACKUP_DIR, { recursive: true });
-
- // Determine container name dynamically if possible, or use a consistent name
- // Using 'docker compose ps' to find the container name for service 'db'
- const containerNameOutput = await $`docker compose ps -q db`.text();
- const containerId = containerNameOutput.trim();
-
- if (!containerId) {
- console.error("❌ Could not find running database container. Is Docker Compose up?");
- process.exit(1);
- }
-
- console.log(`🐳 Found database container ID: ${containerId}`);
- console.log(`📂 Saving backup to: ${FILEPATH}`);
-
- // Execute pg_dump inside the container
- // We use Bun.spawn to pipe stdout directly to a file
- // Note: We avoid passing password via CLI args for security, relying on .pgpass or trust in container,
- // but standard postgres image usually allows 'postgres' user without pass locally or env var.
- // Since we are exec-ing AS the user inside container, auth usually works.
-
- // Using -U (user) and -d (database)
- await $`docker exec -t ${containerId} pg_dump -U ${DB_USER} -d ${DB_DATABASE} --clean --if-exists > ${FILEPATH}`;
-
- console.log("✅ Backup completed successfully!");
-} catch (error) {
- console.error("❌ Backup failed:", error);
- process.exit(1);
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
+
+type DumpOptions = {
+ composeFile: string;
+ service: string;
+ output: string;
+};
+
+function valueAfter(args: string[], index: number, option: string): string {
+ const value = args[index + 1];
+ if (!value) throw new Error(`${option} requires a value`);
+ return value;
+}
+
+function parseOptions(args: string[]): DumpOptions {
+ const timestamp = new Date().toISOString().replaceAll(/[:.]/g, "-");
+ const options: DumpOptions = {
+ composeFile: path.join(repoRoot, "compose.yml"),
+ service: "db",
+ output: path.resolve(process.cwd(), "backups", `backup-${timestamp}.dump`),
+ };
+ for (let index = 0; index < args.length; index += 1) {
+ const argument = args[index];
+ if (argument === "--compose-file") {
+ options.composeFile = path.resolve(
+ valueAfter(args, index, "--compose-file"),
+ );
+ index += 1;
+ } else if (argument === "--service") {
+ options.service = valueAfter(args, index, "--service");
+ index += 1;
+ } else if (argument === "--output") {
+ options.output = path.resolve(valueAfter(args, index, "--output"));
+ index += 1;
+ } else {
+ throw new Error(`Unknown argument: ${argument}`);
+ }
+ }
+ return options;
+}
+
+async function assertAbsent(filePath: string): Promise {
+ try {
+ await access(filePath);
+ throw new Error(`Refusing to overwrite existing file: ${filePath}`);
+ } catch (error) {
+ if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+ return;
+ }
+ throw error;
+ }
}
+
+async function main(): Promise {
+ const options = parseOptions(process.argv.slice(2));
+ const databaseUser = process.env.DB_USER ?? "postgres";
+ const databaseName = process.env.DB_DATABASE ?? "solid-imager";
+ const partialPath = `${options.output}.partial`;
+ await mkdir(path.dirname(options.output), { recursive: true });
+ await assertAbsent(options.output);
+ await assertAbsent(partialPath);
+
+ logger.info(
+ {
+ composeFile: options.composeFile,
+ service: options.service,
+ output: options.output,
+ },
+ "Starting custom-format PostgreSQL dump",
+ );
+ const processHandle = Bun.spawn(
+ [
+ "docker",
+ "compose",
+ "-f",
+ options.composeFile,
+ "exec",
+ "-T",
+ options.service,
+ "pg_dump",
+ "--username",
+ databaseUser,
+ "--dbname",
+ databaseName,
+ "--format=custom",
+ "--no-owner",
+ "--no-privileges",
+ ],
+ {
+ stdout: Bun.file(partialPath),
+ stderr: "pipe",
+ },
+ );
+ const stderr = await new Response(processHandle.stderr).text();
+ const exitCode = await processHandle.exited;
+ if (exitCode !== 0) {
+ await rm(partialPath, { force: true });
+ throw new Error(`pg_dump exited with ${exitCode}: ${stderr.trim()}`);
+ }
+ const outputStat = await stat(partialPath);
+ if (outputStat.size === 0) {
+ await rm(partialPath, { force: true });
+ throw new Error("pg_dump produced an empty file");
+ }
+ await rename(partialPath, options.output);
+ logger.info(
+ { output: options.output, bytes: outputStat.size },
+ "PostgreSQL dump completed",
+ );
+}
+
+main().catch((error: unknown) => {
+ logger.error({ err: error }, "PostgreSQL dump failed");
+ process.exitCode = 1;
+});
diff --git a/apps/server/scripts/lib/ccip-migration-core.ts b/apps/server/scripts/lib/ccip-migration-core.ts
new file mode 100644
index 000000000..055175165
--- /dev/null
+++ b/apps/server/scripts/lib/ccip-migration-core.ts
@@ -0,0 +1,494 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import {
+ access,
+ mkdir,
+ readFile,
+ readdir,
+ realpath,
+ rename,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import path from "node:path";
+import type { CcipVectorRecord } from "@solid-imager/application/ports/ccip-vector-store";
+import { z } from "zod";
+
+export const CCIP_MIGRATION_TOOL_VERSION = "ccip-pgvector-migration-v1";
+export const CCIP_VECTOR_DIMENSIONS = 768;
+
+const uuidSchema = z.string().uuid();
+const dateSchema = z.coerce.date().refine((value) => !Number.isNaN(value.getTime()));
+const vectorRecordSchema = z.object({
+ regionId: z.string().uuid().nullable(),
+ regionKind: z.enum(["full", "person", "manual"]),
+ mediaId: z.string().uuid(),
+ mediaSourceId: z.string().uuid(),
+ vector: z.array(z.number().finite()).length(CCIP_VECTOR_DIMENSIONS),
+ model: z.string().min(1),
+ embeddingVersion: z.number().int().nonnegative(),
+ mediaModifiedAt: dateSchema,
+ inputRevision: z.string().min(1),
+ preprocessingProfile: z.string().min(1),
+ extractedAt: dateSchema,
+});
+
+export type DirectoryManifestEntry = {
+ path: string;
+ bytes: number;
+ sha256: string;
+};
+
+export type DirectoryManifest = {
+ root: string;
+ entries: DirectoryManifestEntry[];
+ totalBytes: number;
+ fingerprint: string;
+};
+
+export type MigrationIssueCode =
+ | "SOURCE_READ_FAILED"
+ | "SOURCE_CHANGED"
+ | "INVALID_RECORD"
+ | "ZERO_NORM_VECTOR"
+ | "CONFLICTING_DUPLICATE"
+ | "SOURCE_ORDER_CHANGED"
+ | "ORPHAN_MEDIA"
+ | "MEDIA_SOURCE_MISMATCH"
+ | "CANONICAL_RECORD_MISSING"
+ | "CHECKPOINT_MISMATCH"
+ | "PARITY_MISMATCH"
+ | "RUST_RERANK_SKIPPED"
+ | "RUST_RERANK_FAILED";
+
+export type MigrationIssue = {
+ code: MigrationIssueCode;
+ message: string;
+ logicalKey?: string;
+ mediaId?: string;
+};
+
+export type ScanSummary = {
+ rawRows: number;
+ uniqueLogicalRows: number;
+ collapsedDuplicates: number;
+ issues: MigrationIssue[];
+};
+
+export type CheckpointIdentity = {
+ sourceFingerprint: string;
+ codeFingerprint: string;
+ schemaFingerprint: string;
+ optionsFingerprint: string;
+};
+
+export type MigrationCheckpoint = CheckpointIdentity & {
+ version: 1;
+ toolVersion: typeof CCIP_MIGRATION_TOOL_VERSION;
+ lastCompletedKey: string | null;
+ completedRecords: number;
+ updatedAt: string;
+};
+
+const checkpointSchema = z.object({
+ version: z.literal(1),
+ toolVersion: z.literal(CCIP_MIGRATION_TOOL_VERSION),
+ sourceFingerprint: z.string().length(64),
+ codeFingerprint: z.string().length(64),
+ schemaFingerprint: z.string().length(64),
+ optionsFingerprint: z.string().length(64),
+ lastCompletedKey: z.string().nullable(),
+ completedRecords: z.number().int().nonnegative(),
+ updatedAt: z.string().datetime(),
+});
+
+export type ExistingMedia = {
+ id: string;
+ mediaSourceId: string;
+};
+
+export function stableJson(value: unknown): string {
+ return JSON.stringify(sortJson(value));
+}
+
+function sortJson(value: unknown): unknown {
+ if (Array.isArray(value)) {
+ return value.map(sortJson);
+ }
+ if (typeof value === "object" && value !== null) {
+ return Object.fromEntries(
+ Object.entries(value)
+ .sort(([left], [right]) => left.localeCompare(right))
+ .map(([key, item]) => [key, sortJson(item)]),
+ );
+ }
+ return value;
+}
+
+export function sha256(value: string | Uint8Array): string {
+ return createHash("sha256").update(value).digest("hex");
+}
+
+async function hashFile(filePath: string): Promise {
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(filePath)) {
+ hash.update(chunk);
+ }
+ return hash.digest("hex");
+}
+
+async function listFiles(root: string, relative = ""): Promise {
+ const directory = path.join(root, relative);
+ const entries = await readdir(directory, { withFileTypes: true });
+ const files: string[] = [];
+ for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+ const child = relative ? path.posix.join(relative, entry.name) : entry.name;
+ if (entry.isSymbolicLink()) {
+ throw new Error(`Symbolic links are not allowed in a CCIP snapshot: ${child}`);
+ }
+ if (entry.isDirectory()) {
+ files.push(...(await listFiles(root, child)));
+ } else if (entry.isFile()) {
+ files.push(child);
+ } else {
+ throw new Error(`Unsupported file type in a CCIP snapshot: ${child}`);
+ }
+ }
+ return files;
+}
+
+export async function createDirectoryManifest(
+ directory: string,
+): Promise {
+ const root = await realpath(directory);
+ const rootStat = await stat(root);
+ if (!rootStat.isDirectory()) {
+ throw new Error(`CCIP source is not a directory: ${root}`);
+ }
+ const entries: DirectoryManifestEntry[] = [];
+ for (const relativePath of await listFiles(root)) {
+ const absolutePath = path.join(root, relativePath);
+ const before = await stat(absolutePath);
+ const fileHash = await hashFile(absolutePath);
+ const after = await stat(absolutePath);
+ if (
+ before.size !== after.size ||
+ before.mtimeMs !== after.mtimeMs ||
+ before.ino !== after.ino
+ ) {
+ throw new Error(`CCIP source changed while hashing: ${relativePath}`);
+ }
+ entries.push({ path: relativePath, bytes: after.size, sha256: fileHash });
+ }
+ const totalBytes = entries.reduce((total, entry) => total + entry.bytes, 0);
+ return {
+ root,
+ entries,
+ totalBytes,
+ fingerprint: sha256(stableJson(entries)),
+ };
+}
+
+export function manifestsMatch(
+ left: DirectoryManifest,
+ right: DirectoryManifest,
+): boolean {
+ return left.fingerprint === right.fingerprint && stableJson(left.entries) === stableJson(right.entries);
+}
+
+export async function filesFingerprint(files: string[]): Promise {
+ const entries: Array<{ path: string; bytes: number; sha256: string }> = [];
+ for (const file of [...files].sort()) {
+ const absolutePath = path.resolve(file);
+ const fileStat = await stat(absolutePath);
+ entries.push({
+ path: absolutePath,
+ bytes: fileStat.size,
+ sha256: await hashFile(absolutePath),
+ });
+ }
+ return sha256(stableJson(entries));
+}
+
+export function sourceLogicalKey(record: CcipVectorRecord): string {
+ return stableJson([
+ record.mediaId,
+ record.model,
+ record.embeddingVersion,
+ record.preprocessingProfile,
+ ]);
+}
+
+export function canonicalLogicalKey(record: CcipVectorRecord): string {
+ if (!record.regionId) {
+ throw new Error(`Canonical CCIP record is missing regionId: ${record.mediaId}`);
+ }
+ return stableJson([
+ record.regionId,
+ record.model,
+ record.embeddingVersion,
+ record.preprocessingProfile,
+ ]);
+}
+
+export function validateRecord(value: unknown): {
+ record?: CcipVectorRecord;
+ issues: MigrationIssue[];
+} {
+ const parsed = vectorRecordSchema.safeParse(value);
+ if (!parsed.success) {
+ return {
+ issues: [
+ {
+ code: "INVALID_RECORD",
+ message: z.prettifyError(parsed.error),
+ mediaId: readStringField(value, "mediaId"),
+ },
+ ],
+ };
+ }
+ const squaredNorm = parsed.data.vector.reduce(
+ (total, component) => total + component * component,
+ 0,
+ );
+ if (!Number.isFinite(squaredNorm) || squaredNorm === 0) {
+ return {
+ issues: [
+ {
+ code: "ZERO_NORM_VECTOR",
+ message: `CCIP vector has zero or non-finite norm: ${parsed.data.mediaId}`,
+ mediaId: parsed.data.mediaId,
+ },
+ ],
+ };
+ }
+ return { record: parsed.data, issues: [] };
+}
+
+function readStringField(value: unknown, field: string): string | undefined {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
+ return undefined;
+ }
+ const fieldValue = Reflect.get(value, field);
+ return typeof fieldValue === "string" ? fieldValue : undefined;
+}
+
+function duplicatePayload(record: CcipVectorRecord): string {
+ return stableJson({
+ regionId: record.regionId,
+ regionKind: record.regionKind,
+ mediaId: record.mediaId,
+ mediaSourceId: record.mediaSourceId,
+ vector: record.vector,
+ model: record.model,
+ embeddingVersion: record.embeddingVersion,
+ mediaModifiedAt: record.mediaModifiedAt.toISOString(),
+ inputRevision: record.inputRevision,
+ preprocessingProfile: record.preprocessingProfile,
+ });
+}
+
+export async function scanCollapsedRecords(
+ batches: AsyncIterable,
+ onRecord?: (record: CcipVectorRecord, logicalKey: string) => Promise,
+): Promise {
+ const summary: ScanSummary = {
+ rawRows: 0,
+ uniqueLogicalRows: 0,
+ collapsedDuplicates: 0,
+ issues: [],
+ };
+ let current:
+ | { key: string; record: CcipVectorRecord; payload: string; conflict: boolean }
+ | undefined;
+ let previousCompletedKey: string | undefined;
+
+ const flush = async () => {
+ if (!current) return;
+ summary.uniqueLogicalRows += 1;
+ if (!current.conflict && onRecord) {
+ await onRecord(current.record, current.key);
+ }
+ previousCompletedKey = current.key;
+ current = undefined;
+ };
+
+ try {
+ for await (const batch of batches) {
+ for (const value of batch) {
+ summary.rawRows += 1;
+ const validation = validateRecord(value);
+ if (!validation.record) {
+ summary.issues.push(...validation.issues);
+ continue;
+ }
+ const record = validation.record;
+ const key = sourceLogicalKey(record);
+ const payload = duplicatePayload(record);
+ if (!current || current.key !== key) {
+ await flush();
+ if (previousCompletedKey && key.localeCompare(previousCompletedKey) < 0) {
+ summary.issues.push({
+ code: "SOURCE_ORDER_CHANGED",
+ message: `Legacy CCIP rows are not ordered deterministically: ${key}`,
+ logicalKey: key,
+ mediaId: record.mediaId,
+ });
+ }
+ current = { key, record, payload, conflict: false };
+ continue;
+ }
+ if (current.payload !== payload) {
+ if (!current.conflict) {
+ summary.issues.push({
+ code: "CONFLICTING_DUPLICATE",
+ message: `Conflicting legacy CCIP records: ${key}`,
+ logicalKey: key,
+ mediaId: record.mediaId,
+ });
+ }
+ current.conflict = true;
+ continue;
+ }
+ summary.collapsedDuplicates += 1;
+ if (record.extractedAt.getTime() > current.record.extractedAt.getTime()) {
+ current.record = record;
+ }
+ }
+ }
+ await flush();
+ } catch (error) {
+ summary.issues.push({
+ code: "SOURCE_READ_FAILED",
+ message: error instanceof Error ? error.message : String(error),
+ });
+ }
+ return summary;
+}
+
+export function validateMediaReferences(
+ records: CcipVectorRecord[],
+ existingMedia: ReadonlyMap,
+): MigrationIssue[] {
+ const issues: MigrationIssue[] = [];
+ for (const record of records) {
+ const media = existingMedia.get(record.mediaId);
+ if (!media) {
+ issues.push({
+ code: "ORPHAN_MEDIA",
+ message: `Legacy CCIP record references missing media: ${record.mediaId}`,
+ logicalKey: sourceLogicalKey(record),
+ mediaId: record.mediaId,
+ });
+ } else if (media.mediaSourceId !== record.mediaSourceId) {
+ issues.push({
+ code: "MEDIA_SOURCE_MISMATCH",
+ message: `Legacy CCIP media source mismatch: ${record.mediaId}`,
+ logicalKey: sourceLogicalKey(record),
+ mediaId: record.mediaId,
+ });
+ }
+ }
+ return issues;
+}
+
+export function createOptionsFingerprint(options: Record): string {
+ return sha256(stableJson(options));
+}
+
+export function createCheckpoint(
+ identity: CheckpointIdentity,
+ lastCompletedKey: string | null,
+ completedRecords: number,
+): MigrationCheckpoint {
+ return {
+ version: 1,
+ toolVersion: CCIP_MIGRATION_TOOL_VERSION,
+ ...identity,
+ lastCompletedKey,
+ completedRecords,
+ updatedAt: new Date().toISOString(),
+ };
+}
+
+export async function readCheckpoint(filePath: string): Promise {
+ return checkpointSchema.parse(JSON.parse(await readFile(filePath, "utf8")));
+}
+
+export function assertCheckpointCompatible(
+ checkpoint: MigrationCheckpoint,
+ identity: CheckpointIdentity,
+): void {
+ for (const field of [
+ "sourceFingerprint",
+ "codeFingerprint",
+ "schemaFingerprint",
+ "optionsFingerprint",
+ ] as const) {
+ if (checkpoint[field] !== identity[field]) {
+ throw new Error(`Checkpoint ${field} does not match this migration run`);
+ }
+ }
+}
+
+export async function assertPathAbsent(filePath: string): Promise {
+ try {
+ await access(filePath);
+ throw new Error(`Refusing to overwrite existing file: ${filePath}`);
+ } catch (error) {
+ if (isNodeError(error) && error.code === "ENOENT") return;
+ throw error;
+ }
+}
+
+function isNodeError(error: unknown): error is NodeJS.ErrnoException {
+ return error instanceof Error && "code" in error;
+}
+
+export async function writeJsonNoOverwrite(
+ filePath: string,
+ value: unknown,
+): Promise {
+ const absolutePath = path.resolve(filePath);
+ const partialPath = `${absolutePath}.partial`;
+ await mkdir(path.dirname(absolutePath), { recursive: true });
+ await assertPathAbsent(absolutePath);
+ await assertPathAbsent(partialPath);
+ try {
+ await writeFile(partialPath, `${JSON.stringify(value, null, 2)}\n`, {
+ flag: "wx",
+ });
+ await rename(partialPath, absolutePath);
+ } catch (error) {
+ await rm(partialPath, { force: true });
+ throw error;
+ }
+}
+
+export async function writeCheckpointAtomic(
+ filePath: string,
+ checkpoint: MigrationCheckpoint,
+ allowReplace: boolean,
+): Promise {
+ const absolutePath = path.resolve(filePath);
+ const partialPath = `${absolutePath}.partial`;
+ await mkdir(path.dirname(absolutePath), { recursive: true });
+ if (!allowReplace) await assertPathAbsent(absolutePath);
+ await assertPathAbsent(partialPath);
+ try {
+ await writeFile(partialPath, `${JSON.stringify(checkpoint, null, 2)}\n`, {
+ flag: "wx",
+ });
+ await rename(partialPath, absolutePath);
+ } catch (error) {
+ await rm(partialPath, { force: true });
+ throw error;
+ }
+}
+
+export function parseUuid(value: string, option: string): string {
+ const result = uuidSchema.safeParse(value);
+ if (!result.success) throw new Error(`${option} requires a UUID`);
+ return result.data;
+}
diff --git a/apps/server/scripts/restore-db.ts b/apps/server/scripts/restore-db.ts
index 7ce20b9a6..1786534f1 100644
--- a/apps/server/scripts/restore-db.ts
+++ b/apps/server/scripts/restore-db.ts
@@ -1,100 +1,220 @@
///
-import { $ } from "bun";
-import { readdir, stat } from "node:fs/promises";
+import { open, stat } from "node:fs/promises";
import path from "node:path";
-
-// Load environment variables
-const DB_USER = process.env.DB_USER || "postgres";
-const DB_DATABASE = process.env.DB_DATABASE || "solid-imager";
-const BACKUP_DIR = "backups";
-
-// Get target backup file from args or find latest
-const args = process.argv.slice(2);
-let targetFile = args[0];
-
-if (!targetFile) {
- try {
- // Check if backup directory exists
- const dirStats = await stat(BACKUP_DIR).catch(() => null);
- if (!dirStats || !dirStats.isDirectory()) {
- console.error(`❌ Backup directory '${BACKUP_DIR}' not found.`);
- process.exit(1);
- }
-
- const files = await readdir(BACKUP_DIR);
- const sqlFiles = files.filter((f) => f.endsWith(".sql"));
-
- if (sqlFiles.length === 0) {
- console.error("❌ No backup files found in backups/ directory.");
- process.exit(1);
- }
-
- // Sort by modification time desc to get the latest
- const fileStats = await Promise.all(
- sqlFiles.map(async (file) => {
- const filePath = path.join(BACKUP_DIR, file);
- const stats = await stat(filePath);
- return { file, mtime: stats.mtime.getTime() };
- }),
- );
-
- fileStats.sort((a, b) => b.mtime - a.mtime);
- targetFile = path.join(BACKUP_DIR, fileStats[0].file);
- console.log(`ℹ️ No file specified. Using latest backup: ${targetFile}`);
- } catch (error) {
- console.error("❌ Error finding backup files:", error);
- process.exit(1);
- }
-} else {
- // Validate provided file exists
- try {
- await stat(targetFile);
- } catch {
- console.error(`❌ Specified backup file not found: ${targetFile}`);
- process.exit(1);
- }
+import { fileURLToPath } from "node:url";
+import { logger } from "../src/infrastructure/logger";
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
+
+type RestoreOptions = {
+ composeFile: string;
+ service: string;
+ input: string;
+ confirmedEmptyTarget: boolean;
+};
+
+function valueAfter(args: string[], index: number, option: string): string {
+ const value = args[index + 1];
+ if (!value) throw new Error(`${option} requires a value`);
+ return value;
}
-console.log(
- `\n⚠️ WARNING: This will OVERWRITE the database '${DB_DATABASE}' with data from '${targetFile}'.`,
-);
-console.log("⚠️ Current data in the database will be lost/modified.");
-console.log("⏳ Starting in 5 seconds... Press Ctrl+C to cancel.");
-
-await new Promise((r) => setTimeout(r, 1000));
-process.stdout.write("5...");
-await new Promise((r) => setTimeout(r, 1000));
-process.stdout.write(" 4...");
-await new Promise((r) => setTimeout(r, 1000));
-process.stdout.write(" 3...");
-await new Promise((r) => setTimeout(r, 1000));
-process.stdout.write(" 2...");
-await new Promise((r) => setTimeout(r, 1000));
-process.stdout.write(" 1...\n");
-
-console.log("📦 Starting database restore...");
-
-try {
- // Find container
- const containerNameOutput = await $`docker compose ps -q db`.text();
- const containerId = containerNameOutput.trim();
-
- if (!containerId) {
- console.error("❌ Could not find running database container. Is Docker Compose up?");
- process.exit(1);
- }
+function parseOptions(args: string[]): RestoreOptions {
+ const options: RestoreOptions = {
+ composeFile: path.join(repoRoot, "compose.yml"),
+ service: "db",
+ input: "",
+ confirmedEmptyTarget: false,
+ };
+ for (let index = 0; index < args.length; index += 1) {
+ const argument = args[index];
+ if (argument === "--compose-file") {
+ options.composeFile = path.resolve(
+ valueAfter(args, index, "--compose-file"),
+ );
+ index += 1;
+ } else if (argument === "--service") {
+ options.service = valueAfter(args, index, "--service");
+ index += 1;
+ } else if (argument === "--input") {
+ options.input = path.resolve(valueAfter(args, index, "--input"));
+ index += 1;
+ } else if (argument === "--confirm-empty-target") {
+ options.confirmedEmptyTarget = true;
+ } else {
+ throw new Error(`Unknown argument: ${argument}`);
+ }
+ }
+ if (!options.input) throw new Error("--input is required");
+ if (!options.confirmedEmptyTarget) {
+ throw new Error("--confirm-empty-target is required for restore");
+ }
+ return options;
+}
- console.log(`🐳 Found database container ID: ${containerId}`);
+function composeCommand(options: RestoreOptions, command: string[]): string[] {
+ return [
+ "docker",
+ "compose",
+ "-f",
+ options.composeFile,
+ "exec",
+ "-T",
+ options.service,
+ ...command,
+ ];
+}
- // Execute restore
- // We use Bun.file to read the SQL file and pipe it into the docker exec command
- const fileInput = Bun.file(targetFile);
+async function runCapture(command: string[]): Promise {
+ const processHandle = Bun.spawn(command, { stdout: "pipe", stderr: "pipe" });
+ const [stdout, stderr, exitCode] = await Promise.all([
+ new Response(processHandle.stdout).text(),
+ new Response(processHandle.stderr).text(),
+ processHandle.exited,
+ ]);
+ if (exitCode !== 0) {
+ throw new Error(`${command[0]} exited with ${exitCode}: ${stderr.trim()}`);
+ }
+ return stdout.trim();
+}
- // Note: -i is required for docker exec to accept stdin
- await $`docker exec -i ${containerId} psql -U ${DB_USER} -d ${DB_DATABASE} < ${fileInput}`;
+async function isCustomDump(filePath: string): Promise {
+ const handle = await open(filePath, "r");
+ try {
+ const signature = Buffer.alloc(5);
+ await handle.read(signature, 0, signature.length, 0);
+ return signature.toString("ascii") === "PGDMP";
+ } finally {
+ await handle.close();
+ }
+}
- console.log("✅ Restore completed successfully!");
-} catch (error) {
- console.error("❌ Restore failed:", error);
- process.exit(1);
+async function main(): Promise {
+ const options = parseOptions(process.argv.slice(2));
+ const databaseUser = process.env.DB_USER ?? "postgres";
+ const databaseName = process.env.DB_DATABASE ?? "solid-imager";
+ const inputStat = await stat(options.input);
+ if (!inputStat.isFile() || inputStat.size === 0) {
+ throw new Error(`Restore input must be a non-empty file: ${options.input}`);
+ }
+
+ const objectCountOutput = await runCapture(
+ composeCommand(options, [
+ "psql",
+ "-X",
+ "--username",
+ databaseUser,
+ "--dbname",
+ databaseName,
+ "--tuples-only",
+ "--no-align",
+ "--command",
+ `WITH user_objects AS (
+ SELECT class.oid
+ FROM pg_catalog.pg_class class
+ INNER JOIN pg_catalog.pg_namespace namespace ON namespace.oid = class.relnamespace
+ WHERE class.relkind IN ('r', 'p', 'S', 'v', 'm', 'f')
+ AND namespace.nspname = 'public'
+ AND NOT EXISTS (
+ SELECT 1 FROM pg_catalog.pg_depend dependency
+ WHERE dependency.classid = 'pg_class'::regclass
+ AND dependency.objid = class.oid
+ AND dependency.deptype = 'e'
+ )
+ UNION ALL
+ SELECT type.oid
+ FROM pg_catalog.pg_type type
+ INNER JOIN pg_catalog.pg_namespace namespace ON namespace.oid = type.typnamespace
+ WHERE type.typtype IN ('e', 'd')
+ AND namespace.nspname = 'public'
+ AND NOT EXISTS (
+ SELECT 1 FROM pg_catalog.pg_depend dependency
+ WHERE dependency.classid = 'pg_type'::regclass
+ AND dependency.objid = type.oid
+ AND dependency.deptype = 'e'
+ )
+ UNION ALL
+ SELECT procedure.oid
+ FROM pg_catalog.pg_proc procedure
+ INNER JOIN pg_catalog.pg_namespace namespace ON namespace.oid = procedure.pronamespace
+ WHERE namespace.nspname = 'public'
+ AND NOT EXISTS (
+ SELECT 1 FROM pg_catalog.pg_depend dependency
+ WHERE dependency.classid = 'pg_proc'::regclass
+ AND dependency.objid = procedure.oid
+ AND dependency.deptype = 'e'
+ )
+ UNION ALL
+ SELECT namespace.oid
+ FROM pg_catalog.pg_namespace namespace
+ WHERE namespace.nspname NOT IN ('public', 'pg_catalog', 'information_schema')
+ AND namespace.nspname !~ '^pg_'
+ )
+ SELECT count(*) FROM user_objects;`,
+ ]),
+ );
+ const objectCount = Number.parseInt(objectCountOutput, 10);
+ if (!Number.isSafeInteger(objectCount) || objectCount !== 0) {
+ throw new Error(
+ `Restore target is not empty (${objectCountOutput || "unknown"} user objects)`,
+ );
+ }
+
+ const custom = await isCustomDump(options.input);
+ const restoreCommand = custom
+ ? [
+ "pg_restore",
+ "--exit-on-error",
+ "--clean",
+ "--if-exists",
+ "--no-owner",
+ "--no-privileges",
+ "--username",
+ databaseUser,
+ "--dbname",
+ databaseName,
+ ]
+ : [
+ "psql",
+ "-X",
+ "--set=ON_ERROR_STOP=1",
+ "--username",
+ databaseUser,
+ "--dbname",
+ databaseName,
+ ];
+ logger.info(
+ {
+ input: options.input,
+ format: custom ? "custom" : "plain",
+ composeFile: options.composeFile,
+ service: options.service,
+ },
+ "Starting PostgreSQL restore into verified empty target",
+ );
+ const processHandle = Bun.spawn(composeCommand(options, restoreCommand), {
+ stdin: Bun.file(options.input).stream(),
+ stdout: "pipe",
+ stderr: "pipe",
+ });
+ const [stdout, stderr, exitCode] = await Promise.all([
+ new Response(processHandle.stdout).text(),
+ new Response(processHandle.stderr).text(),
+ processHandle.exited,
+ ]);
+ void stdout;
+ if (exitCode !== 0) {
+ throw new Error(
+ `${custom ? "pg_restore" : "psql"} exited with ${exitCode}: ${stderr.trim()}`,
+ );
+ }
+ logger.info(
+ { input: options.input, bytes: inputStat.size },
+ "PostgreSQL restore completed",
+ );
}
+
+main().catch((error: unknown) => {
+ logger.error({ err: error }, "PostgreSQL restore failed");
+ process.exitCode = 1;
+});
diff --git a/apps/server/scripts/set-jobs-logged.ts b/apps/server/scripts/set-jobs-logged.ts
new file mode 100644
index 000000000..255bec728
--- /dev/null
+++ b/apps/server/scripts/set-jobs-logged.ts
@@ -0,0 +1,173 @@
+import { sql } from "drizzle-orm";
+import { z } from "zod";
+import type { DrizzleExecutor } from "@solid-imager/db/types";
+import { db } from "../src/infrastructure/db";
+import { logger } from "../src/infrastructure/logger";
+
+const auditRowSchema = z.object({
+ relpersistence: z.enum(["p", "u"]),
+ totalJobs: z.coerce.number().int().nonnegative(),
+ tableBytes: z.coerce.number().int().nonnegative(),
+ indexBytes: z.coerce.number().int().nonnegative(),
+ totalBytes: z.coerce.number().int().nonnegative(),
+ inProgressJobs: z.coerce.number().int().nonnegative(),
+ missingQueueNames: z.coerce.number().int().nonnegative(),
+ orphanParents: z.coerce.number().int().nonnegative(),
+ duplicateActiveDedupeKeys: z.coerce.number().int().nonnegative(),
+ duplicateRunningConcurrencyKeys: z.coerce.number().int().nonnegative(),
+ invalidRetryRows: z.coerce.number().int().nonnegative(),
+});
+
+type Audit = z.infer;
+
+const maxLockWaitMs = 5_000;
+
+function isRecord(value: unknown): value is Record {
+ return typeof value === "object" && value !== null;
+}
+
+function firstRow(value: unknown): unknown {
+ if (Array.isArray(value)) return value[0];
+ if (isRecord(value) && Array.isArray(value.rows)) return value.rows[0];
+ return undefined;
+}
+
+async function auditJobs(
+ executor: Pick = db,
+): Promise {
+ const result = await executor.execute(sql`
+ SELECT
+ class.relpersistence AS "relpersistence",
+ (SELECT count(*) FROM jobs) AS "totalJobs",
+ pg_relation_size(class.oid) AS "tableBytes",
+ pg_indexes_size(class.oid) AS "indexBytes",
+ pg_total_relation_size(class.oid) AS "totalBytes",
+ (SELECT count(*) FROM jobs WHERE status = 'in_progress') AS "inProgressJobs",
+ (SELECT count(*) FROM jobs WHERE queue_name IS NULL) AS "missingQueueNames",
+ (SELECT count(*) FROM jobs child LEFT JOIN jobs parent ON parent.id = child.parent_id WHERE child.parent_id IS NOT NULL AND parent.id IS NULL) AS "orphanParents",
+ (SELECT count(*) FROM (SELECT dedupe_key FROM jobs WHERE dedupe_key IS NOT NULL AND status IN ('pending', 'in_progress') GROUP BY dedupe_key HAVING count(*) > 1) duplicates) AS "duplicateActiveDedupeKeys",
+ (SELECT count(*) FROM (SELECT concurrency_key FROM jobs WHERE concurrency_key IS NOT NULL AND status = 'in_progress' GROUP BY concurrency_key HAVING count(*) > 1) duplicates) AS "duplicateRunningConcurrencyKeys",
+ (SELECT count(*) FROM jobs WHERE attempt_count < 0 OR max_attempts <= 0 OR lease_duration_ms <= 0) AS "invalidRetryRows"
+ FROM pg_class class
+ INNER JOIN pg_namespace namespace ON namespace.oid = class.relnamespace
+ WHERE class.relname = 'jobs' AND namespace.nspname = current_schema()
+ `);
+ return auditRowSchema.parse(firstRow(result));
+}
+
+function assertReadyForRewrite(audit: Audit): void {
+ if (audit.inProgressJobs > 0) {
+ throw new Error(
+ `Jobs are not quiesced: ${audit.inProgressJobs} job(s) are in_progress`,
+ );
+ }
+ const invalidRows =
+ audit.missingQueueNames +
+ audit.orphanParents +
+ audit.duplicateActiveDedupeKeys +
+ audit.duplicateRunningConcurrencyKeys +
+ audit.invalidRetryRows;
+ if (invalidRows > 0) {
+ throw new Error(
+ `Jobs validation failed with ${invalidRows} row/group violation(s)`,
+ );
+ }
+}
+
+function isReadyForRewrite(audit: Audit): boolean {
+ try {
+ assertReadyForRewrite(audit);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function main(): Promise {
+ const startedAt = new Date();
+ const startedAtMs = Date.now();
+ const args = new Set(process.argv.slice(2));
+ const apply = args.has("--apply");
+ if (args.size > (apply ? 2 : 0)) {
+ throw new Error(
+ "Usage: bun scripts/set-jobs-logged.ts [--apply --confirm-jobs-quiesced]",
+ );
+ }
+ if (process.env.DB_HOST === "pglite") {
+ throw new Error("SET LOGGED maintenance is only valid for PostgreSQL");
+ }
+ const before = await auditJobs();
+ if (!apply) {
+ const finishedAt = new Date();
+ process.stdout.write(
+ `${JSON.stringify(
+ {
+ mode: "dry-run",
+ ready: isReadyForRewrite(before),
+ startedAt: startedAt.toISOString(),
+ finishedAt: finishedAt.toISOString(),
+ elapsedMs: Date.now() - startedAtMs,
+ maxLockWaitMs,
+ before,
+ },
+ null,
+ 2,
+ )}\n`,
+ );
+ return;
+ }
+ if (!args.has("--confirm-jobs-quiesced")) {
+ throw new Error("--apply requires --confirm-jobs-quiesced");
+ }
+ assertReadyForRewrite(before);
+ let rewriteElapsedMs = 0;
+ if (before.relpersistence === "u") {
+ const rewriteStartedAtMs = Date.now();
+ await db.transaction(async (transaction) => {
+ await transaction.execute(sql`SET LOCAL lock_timeout = '5s'`);
+ await transaction.execute(
+ sql`SELECT pg_advisory_xact_lock(hashtext('solid-imager.jobs.set-logged'))`,
+ );
+ assertReadyForRewrite(await auditJobs(transaction));
+ await transaction.execute(sql`ALTER TABLE jobs SET LOGGED`);
+ });
+ rewriteElapsedMs = Date.now() - rewriteStartedAtMs;
+ }
+ const after = await auditJobs();
+ if (after.relpersistence !== "p") {
+ throw new Error("jobs relpersistence did not become permanent");
+ }
+ logger.info(
+ {
+ totalJobs: after.totalJobs,
+ tableBytes: after.tableBytes,
+ indexBytes: after.indexBytes,
+ totalBytes: after.totalBytes,
+ rewriteElapsedMs,
+ },
+ "jobs table is WAL-logged and validated",
+ );
+ const finishedAt = new Date();
+ process.stdout.write(
+ `${JSON.stringify(
+ {
+ mode: "apply",
+ changed: before.relpersistence === "u",
+ startedAt: startedAt.toISOString(),
+ finishedAt: finishedAt.toISOString(),
+ elapsedMs: Date.now() - startedAtMs,
+ rewriteElapsedMs,
+ maxLockWaitMs,
+ before,
+ after,
+ },
+ null,
+ 2,
+ )}\n`,
+ );
+}
+
+main().catch((error: unknown) => {
+ logger.error({ err: error }, "SET LOGGED maintenance failed");
+ process.exitCode = 1;
+});
diff --git a/apps/server/scripts/validate-postgres-rehearsal.ts b/apps/server/scripts/validate-postgres-rehearsal.ts
new file mode 100644
index 000000000..49eb3d297
--- /dev/null
+++ b/apps/server/scripts/validate-postgres-rehearsal.ts
@@ -0,0 +1,404 @@
+///
+import { readFile, rename, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { z } from "zod";
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
+
+const reportSchema = z.object({
+ ok: z.boolean(),
+ serverMajor: z.number().int(),
+ serverVersionNum: z.number().int(),
+ serverVersion: z.string(),
+ vectorAvailable: z.boolean(),
+ vectorVersion: z.string().nullable(),
+ migrations: z.array(
+ z.object({
+ id: z.number().int(),
+ hash: z.string(),
+ createdAt: z.number().int(),
+ }),
+ ),
+ constraints: z.array(
+ z.object({
+ name: z.string(),
+ type: z.string(),
+ definition: z.string(),
+ validated: z.boolean(),
+ }),
+ ),
+ invalidConstraintCount: z.number().int().nonnegative(),
+ tableCounts: z.record(z.string(), z.number().int().nonnegative()),
+ readWriteProbe: z.literal(true),
+ vectorProbe: z.boolean(),
+ mismatches: z.array(z.string()),
+});
+
+type ValidationReport = z.infer;
+type Options = {
+ composeFile: string;
+ service: string;
+ expectedReport?: string;
+ output?: string;
+ expectedMajor: number;
+ expectVectorAvailable: boolean;
+ expectedVectorVersion?: string;
+ allowedAddedTableCounts: Map;
+};
+
+const defaultAllowedAddedTableCounts = new Map([
+ ["ccip_embeddings", 0],
+ ["media_regions", 0],
+]);
+
+function valueAfter(args: string[], index: number, option: string): string {
+ const value = args[index + 1];
+ if (!value) throw new Error(`${option} requires a value`);
+ return value;
+}
+
+function parseOptions(args: string[]): Options {
+ const options: Options = {
+ composeFile: path.join(repoRoot, "compose.pg18-rehearsal.yml"),
+ service: "db-pg18-rehearsal",
+ expectedMajor: 18,
+ expectVectorAvailable: true,
+ expectedVectorVersion: "0.8.5",
+ allowedAddedTableCounts: new Map(defaultAllowedAddedTableCounts),
+ };
+ for (let index = 0; index < args.length; index += 1) {
+ const argument = args[index];
+ if (argument === "--compose-file") {
+ options.composeFile = path.resolve(
+ valueAfter(args, index, "--compose-file"),
+ );
+ index += 1;
+ } else if (argument === "--service") {
+ options.service = valueAfter(args, index, "--service");
+ index += 1;
+ } else if (argument === "--expected-report") {
+ options.expectedReport = path.resolve(
+ valueAfter(args, index, "--expected-report"),
+ );
+ index += 1;
+ } else if (argument === "--output") {
+ options.output = path.resolve(valueAfter(args, index, "--output"));
+ index += 1;
+ } else if (argument === "--expected-major") {
+ const value = Number.parseInt(
+ valueAfter(args, index, "--expected-major"),
+ 10,
+ );
+ if (!Number.isSafeInteger(value) || value < 10) {
+ throw new Error("--expected-major must be a PostgreSQL major version");
+ }
+ options.expectedMajor = value;
+ index += 1;
+ } else if (argument === "--expected-vector-version") {
+ options.expectedVectorVersion = valueAfter(
+ args,
+ index,
+ "--expected-vector-version",
+ );
+ index += 1;
+ } else if (argument === "--allow-any-vector-version") {
+ options.expectVectorAvailable = true;
+ options.expectedVectorVersion = undefined;
+ } else if (argument === "--expect-vector-unavailable") {
+ options.expectVectorAvailable = false;
+ options.expectedVectorVersion = undefined;
+ } else if (argument === "--allow-added-table") {
+ const value = valueAfter(args, index, "--allow-added-table");
+ const match = /^([a-z][a-z0-9_]*)=(\d+)$/.exec(value);
+ if (!match) {
+ throw new Error(
+ "--allow-added-table must use the form table_name=expected_count",
+ );
+ }
+ options.allowedAddedTableCounts.set(
+ match[1],
+ parseInteger(match[2], `${match[1]} expected row count`),
+ );
+ index += 1;
+ } else {
+ throw new Error(`Unknown argument: ${argument}`);
+ }
+ }
+ return options;
+}
+
+async function query(options: Options, sql: string): Promise {
+ const databaseUser = process.env.DB_USER ?? "postgres";
+ const databaseName = process.env.DB_DATABASE ?? "solid-imager";
+ const processHandle = Bun.spawn(
+ [
+ "docker",
+ "compose",
+ "-f",
+ options.composeFile,
+ "exec",
+ "-T",
+ options.service,
+ "psql",
+ "-X",
+ "--set=ON_ERROR_STOP=1",
+ "--username",
+ databaseUser,
+ "--dbname",
+ databaseName,
+ "--tuples-only",
+ "--no-align",
+ "--command",
+ sql,
+ ],
+ { stdout: "pipe", stderr: "pipe" },
+ );
+ const [stdout, stderr, exitCode] = await Promise.all([
+ new Response(processHandle.stdout).text(),
+ new Response(processHandle.stderr).text(),
+ processHandle.exited,
+ ]);
+ if (exitCode !== 0) {
+ throw new Error(`psql exited with ${exitCode}: ${stderr.trim()}`);
+ }
+ return stdout.trim();
+}
+
+function parseInteger(value: string, name: string): number {
+ const parsed = Number.parseInt(value, 10);
+ if (!Number.isSafeInteger(parsed) || parsed < 0) {
+ throw new Error(`Invalid ${name}: ${value}`);
+ }
+ return parsed;
+}
+
+async function collectReport(options: Options): Promise {
+ await query(options, "ANALYZE;");
+ const metadata = z
+ .object({
+ serverVersionNum: z.number().int(),
+ serverVersion: z.string(),
+ vectorVersion: z.string().nullable(),
+ invalidConstraintCount: z.number().int(),
+ })
+ .parse(
+ JSON.parse(
+ await query(
+ options,
+ `SELECT json_build_object(
+ 'serverVersionNum', current_setting('server_version_num')::integer,
+ 'serverVersion', version(),
+ 'vectorVersion', (SELECT extversion FROM pg_extension WHERE extname = 'vector'),
+ 'invalidConstraintCount', (SELECT count(*)::integer FROM pg_constraint WHERE NOT convalidated)
+ )::text;`,
+ ),
+ ),
+ );
+ const tableNamesOutput = await query(
+ options,
+ "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename;",
+ );
+ const tableNames = tableNamesOutput ? tableNamesOutput.split("\n") : [];
+ const tableCounts: Record = {};
+ for (const tableName of tableNames) {
+ if (!/^[a-z][a-z0-9_]*$/.test(tableName)) {
+ throw new Error(`Unsafe table name returned by PostgreSQL: ${tableName}`);
+ }
+ tableCounts[tableName] = parseInteger(
+ await query(options, `SELECT count(*) FROM public."${tableName}";`),
+ `${tableName} row count`,
+ );
+ }
+ const migrations = z
+ .array(
+ z.object({
+ id: z.number().int(),
+ hash: z.string(),
+ createdAt: z.number().int(),
+ }),
+ )
+ .parse(
+ JSON.parse(
+ await query(
+ options,
+ "SELECT coalesce(json_agg(json_build_object('id', id, 'hash', hash, 'createdAt', created_at) ORDER BY created_at), '[]'::json)::text FROM drizzle.__drizzle_migrations;",
+ ),
+ ),
+ );
+ const constraints = z
+ .array(
+ z.object({
+ name: z.string(),
+ type: z.string(),
+ definition: z.string(),
+ validated: z.boolean(),
+ }),
+ )
+ .parse(
+ JSON.parse(
+ await query(
+ options,
+ `SELECT coalesce(json_agg(
+ json_build_object(
+ 'name', constraint.conname,
+ 'type', constraint.contype::text,
+ 'definition', regexp_replace(pg_get_constraintdef(constraint.oid, true), '\\s+', ' ', 'g'),
+ 'validated', constraint.convalidated
+ )
+ ORDER BY constraint.conname
+ ), '[]'::json)::text
+ FROM pg_constraint constraint
+ INNER JOIN pg_namespace namespace ON namespace.oid = constraint.connamespace
+ WHERE namespace.nspname = 'public';`,
+ ),
+ ),
+ );
+ const readWriteProbe =
+ (await query(
+ options,
+ "BEGIN; CREATE TEMP TABLE pg18_rehearsal_probe(value integer); INSERT INTO pg18_rehearsal_probe VALUES (1); SELECT count(*) FROM pg18_rehearsal_probe; ROLLBACK;",
+ )).includes("1");
+ const vectorAvailable = metadata.vectorVersion !== null;
+ const vectorProbe = vectorAvailable
+ ? (await query(
+ options,
+ "SELECT ('[1,0,0]'::vector <=> '[1,0,0]'::vector) = 0;",
+ )) === "t"
+ : false;
+ if (!readWriteProbe) {
+ throw new Error("PostgreSQL read/write validation failed");
+ }
+ const serverMajor = Math.floor(metadata.serverVersionNum / 10_000);
+ const mismatches: string[] = [];
+ if (serverMajor !== options.expectedMajor) {
+ mismatches.push(
+ `PostgreSQL major: expected ${options.expectedMajor}, got ${serverMajor}`,
+ );
+ }
+ if (vectorAvailable !== options.expectVectorAvailable) {
+ mismatches.push(
+ `vector extension availability: expected ${options.expectVectorAvailable}, got ${vectorAvailable}`,
+ );
+ }
+ if (vectorAvailable && !vectorProbe) {
+ mismatches.push("vector extension probe failed");
+ }
+ if (
+ options.expectVectorAvailable &&
+ options.expectedVectorVersion &&
+ metadata.vectorVersion !== options.expectedVectorVersion
+ ) {
+ mismatches.push(
+ `vector extension: expected ${options.expectedVectorVersion}, got ${metadata.vectorVersion}`,
+ );
+ }
+ if (metadata.invalidConstraintCount !== 0) {
+ mismatches.push(
+ `invalid constraints: expected 0, got ${metadata.invalidConstraintCount}`,
+ );
+ }
+ return {
+ ok: mismatches.length === 0,
+ serverMajor,
+ ...metadata,
+ vectorAvailable,
+ migrations,
+ constraints,
+ tableCounts,
+ readWriteProbe: true,
+ vectorProbe,
+ mismatches,
+ };
+}
+
+function compareReports(
+ report: ValidationReport,
+ expected: ValidationReport,
+ allowedAddedTableCounts: ReadonlyMap,
+): string[] {
+ const mismatches: string[] = [];
+ const expectedTableNames = Object.keys(expected.tableCounts).sort();
+ const actualTableNames = Object.keys(report.tableCounts).sort();
+ const unexpectedAddedTableNames = actualTableNames.filter(
+ (tableName) =>
+ !expectedTableNames.includes(tableName) &&
+ !allowedAddedTableCounts.has(tableName),
+ );
+ if (unexpectedAddedTableNames.length > 0) {
+ mismatches.push(
+ `unexpected target-only tables: ${unexpectedAddedTableNames.join(",")}`,
+ );
+ }
+ for (const [tableName, expectedCount] of Object.entries(expected.tableCounts)) {
+ const actualCount = report.tableCounts[tableName];
+ if (actualCount !== expectedCount) {
+ mismatches.push(
+ `table ${tableName}: expected ${expectedCount}, got ${actualCount ?? "missing"}`,
+ );
+ }
+ }
+ for (const [tableName, expectedCount] of allowedAddedTableCounts) {
+ if (Object.hasOwn(expected.tableCounts, tableName)) continue;
+ const actualCount = report.tableCounts[tableName];
+ if (actualCount !== expectedCount) {
+ mismatches.push(
+ `target-only table ${tableName}: expected ${expectedCount}, got ${actualCount ?? "missing"}`,
+ );
+ }
+ }
+ for (let index = 0; index < expected.migrations.length; index += 1) {
+ const expectedMigration = expected.migrations[index];
+ const actualMigration = report.migrations[index];
+ if (
+ !actualMigration ||
+ actualMigration.id !== expectedMigration.id ||
+ actualMigration.hash !== expectedMigration.hash
+ ) {
+ mismatches.push(
+ `migration prefix mismatch at position ${index}: expected ${expectedMigration.id}/${expectedMigration.hash}, got ${actualMigration ? `${actualMigration.id}/${actualMigration.hash}` : "missing"}`,
+ );
+ }
+ }
+ const actualConstraints = new Map(
+ report.constraints.map((constraint) => [constraint.name, constraint]),
+ );
+ for (const expectedConstraint of expected.constraints) {
+ const actualConstraint = actualConstraints.get(expectedConstraint.name);
+ if (
+ !actualConstraint ||
+ actualConstraint.type !== expectedConstraint.type ||
+ actualConstraint.definition !== expectedConstraint.definition ||
+ actualConstraint.validated !== expectedConstraint.validated
+ ) {
+ mismatches.push(`constraint mismatch: ${expectedConstraint.name}`);
+ }
+ }
+ return mismatches;
+}
+
+async function main(): Promise {
+ const options = parseOptions(process.argv.slice(2));
+ const report = await collectReport(options);
+ if (options.expectedReport) {
+ const expected = reportSchema.parse(
+ JSON.parse(await readFile(options.expectedReport, "utf8")),
+ );
+ report.mismatches = [
+ ...report.mismatches,
+ ...compareReports(report, expected, options.allowedAddedTableCounts),
+ ];
+ report.ok = report.mismatches.length === 0;
+ }
+ const output = `${JSON.stringify(report, null, 2)}\n`;
+ if (options.output) {
+ const partial = `${options.output}.partial`;
+ await writeFile(partial, output, { flag: "wx" });
+ await rename(partial, options.output);
+ } else {
+ process.stdout.write(output);
+ }
+ if (!report.ok) process.exitCode = 1;
+}
+
+await main();
diff --git a/apps/server/scripts/verify-pglite-bundle.ts b/apps/server/scripts/verify-pglite-bundle.ts
new file mode 100644
index 000000000..fb2c54a43
--- /dev/null
+++ b/apps/server/scripts/verify-pglite-bundle.ts
@@ -0,0 +1,128 @@
+import { cp, mkdtemp, readFile, readdir, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+type PgliteLike = {
+ exec(sql: string): Promise;
+ query(sql: string): Promise<{ rows: T[] }>;
+ close(): Promise;
+};
+
+type PgliteConstructor = new (
+ dataDir: string,
+ options: { extensions: { vector: unknown } },
+) => PgliteLike;
+
+type MigrationJournal = {
+ entries: Array<{ tag: string }>;
+};
+
+function isRecord(value: unknown): value is Record {
+ return typeof value === "object" && value !== null;
+}
+
+function findChunk(files: string[], packageName: string): string {
+ const match = files.find(
+ (file) =>
+ file.endsWith(".mjs") &&
+ file.includes(packageName) &&
+ (packageName.includes("pgvector") || !file.includes("pgvector")),
+ );
+ if (!match) {
+ throw new Error(`Bundled module chunk not found for ${packageName}`);
+ }
+ return match;
+}
+
+async function loadBundledRuntime(libsDir: string): Promise<{
+ PGlite: PgliteConstructor;
+ vector: unknown;
+}> {
+ const files = await readdir(libsDir);
+ const pglitePath = path.join(libsDir, findChunk(files, "electric-sql__pglite"));
+ const vectorPath = path.join(
+ libsDir,
+ findChunk(files, "electric-sql__pglite-pgvector"),
+ );
+ const pgliteModule: unknown = await import(pathToFileURL(pglitePath).href);
+ const vectorModule: unknown = await import(pathToFileURL(vectorPath).href);
+ if (
+ !isRecord(pgliteModule) ||
+ typeof pgliteModule.PGlite !== "function" ||
+ !isRecord(vectorModule) ||
+ !("vector" in vectorModule)
+ ) {
+ throw new Error("Bundled PGlite modules do not expose the expected API");
+ }
+ return {
+ PGlite: pgliteModule.PGlite as PgliteConstructor,
+ vector: vectorModule.vector,
+ };
+}
+
+async function main(): Promise {
+ const outputServer = path.resolve(process.cwd(), ".output/server");
+ const sandbox = await mkdtemp(
+ path.join(os.tmpdir(), "solid-imager-pglite-bundle-"),
+ );
+ try {
+ const isolatedServer = path.join(sandbox, "server");
+ await cp(outputServer, isolatedServer, { recursive: true });
+ const libsDir = path.join(isolatedServer, "_libs");
+ const { PGlite, vector } = await loadBundledRuntime(libsDir);
+ const dataDir = path.join(sandbox, "database");
+ let database = new PGlite(dataDir, { extensions: { vector } });
+ const migrationsDir = path.join(isolatedServer, "drizzle");
+ const journal: MigrationJournal = JSON.parse(
+ await readFile(path.join(migrationsDir, "meta", "_journal.json"), "utf8"),
+ );
+ for (const entry of journal.entries) {
+ const migrationSql = await readFile(
+ path.join(migrationsDir, `${entry.tag}.sql`),
+ "utf8",
+ );
+ if (!migrationSql.trim()) {
+ throw new Error(`Bundled migration is empty: ${entry.tag}`);
+ }
+ for (const statement of migrationSql.split("--> statement-breakpoint")) {
+ if (statement.trim()) await database.exec(statement);
+ }
+ }
+ await database.exec(`
+ CREATE TABLE bundle_probe (id integer PRIMARY KEY, embedding vector(3));
+ INSERT INTO bundle_probe VALUES (1, '[1,0,0]');
+ `);
+ await database.close();
+
+ database = new PGlite(dataDir, { extensions: { vector } });
+ const result = await database.query<{
+ count: number;
+ migrationCount: number;
+ vectorInstalled: boolean;
+ }>(`
+ SELECT
+ (SELECT count(*)::integer FROM bundle_probe WHERE embedding <=> '[1,0,0]'::vector = 0) AS count,
+ ${journal.entries.length}::integer AS "migrationCount",
+ EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') AS "vectorInstalled"
+ `);
+ await database.close();
+ const row = result.rows[0];
+ if (
+ row?.count !== 1 ||
+ row.migrationCount !== journal.entries.length ||
+ !row.vectorInstalled
+ ) {
+ throw new Error(
+ "Bundled PGlite migrations/vector database did not survive close/reopen",
+ );
+ }
+ process.stdout.write(
+ `${JSON.stringify({ ok: true, runtime: "isolated-output", vector: true, migrations: journal.entries.length })}\n`,
+ );
+ } finally {
+ await rm(sandbox, { recursive: true, force: true });
+ }
+}
+
+await main();
diff --git a/apps/server/src/application/registry.ts b/apps/server/src/application/registry.ts
index b09ed7034..616419c0c 100644
--- a/apps/server/src/application/registry.ts
+++ b/apps/server/src/application/registry.ts
@@ -1,3 +1,4 @@
+import type { MediaRegionService } from "@solid-imager/application/services/media-region-service";
import type {
IConfigService,
IFileSystem,
@@ -8,6 +9,7 @@ import type { IAuthorRepository } from "@solid-imager/core/domain/repositories/a
import type { CharacterRepository } from "@solid-imager/core/domain/repositories/character-repository";
import type { IIpRepository } from "@solid-imager/core/domain/repositories/ip-repository";
import type { IJobRepository } from "@solid-imager/core/domain/repositories/job-repository";
+import type { IMediaRegionRepository } from "@solid-imager/core/domain/repositories/media-region-repository";
import type { IMediaRepository } from "@solid-imager/core/domain/repositories/media-repository";
import type { IProjectRepository } from "@solid-imager/core/domain/repositories/project-repository";
import type { SourceRepository } from "@solid-imager/core/domain/repositories/source-repository";
@@ -20,6 +22,8 @@ import type { JobWorker } from "~/infrastructure/jobs/job-worker";
export class ServiceRegistry {
private static instance: ServiceRegistry;
private mediaRepository?: IMediaRepository;
+ private mediaRegionRepository?: IMediaRegionRepository;
+ private mediaRegionService?: MediaRegionService;
private sourceRepository?: SourceRepository;
private mediaStorage?: IMediaStorage;
private fileSystem?: IFileSystem;
@@ -49,6 +53,14 @@ export class ServiceRegistry {
this.mediaRepository = repo;
}
+ registerMediaRegionRepository(repo: IMediaRegionRepository): void {
+ this.mediaRegionRepository = repo;
+ }
+
+ registerMediaRegionService(service: MediaRegionService): void {
+ this.mediaRegionService = service;
+ }
+
registerSourceRepository(repo: SourceRepository): void {
this.sourceRepository = repo;
}
@@ -104,6 +116,20 @@ export class ServiceRegistry {
return this.mediaRepository;
}
+ getMediaRegionRepository(): IMediaRegionRepository {
+ if (!this.mediaRegionRepository) {
+ throw new Error("MediaRegionRepository has not been registered.");
+ }
+ return this.mediaRegionRepository;
+ }
+
+ getMediaRegionService(): MediaRegionService {
+ if (!this.mediaRegionService) {
+ throw new Error("MediaRegionService has not been registered.");
+ }
+ return this.mediaRegionService;
+ }
+
getSourceRepository(): SourceRepository {
if (!this.sourceRepository) {
throw new Error("SourceRepository has not been registered.");
@@ -228,6 +254,8 @@ export class ServiceRegistry {
// Helper for testing to reset the registry
async reset(): Promise {
this.mediaRepository = undefined;
+ this.mediaRegionRepository = undefined;
+ this.mediaRegionService = undefined;
this.sourceRepository = undefined;
this.mediaStorage = undefined;
this.fileSystem = undefined;
diff --git a/apps/server/src/application/services/backup-service.ts b/apps/server/src/application/services/backup-service.ts
index 3e031da52..c03b6a9cc 100644
--- a/apps/server/src/application/services/backup-service.ts
+++ b/apps/server/src/application/services/backup-service.ts
@@ -5,10 +5,11 @@ import {
type MediaDumpItem,
mediaDumpItemSchema,
} from "@solid-imager/core/domain/media/schemas";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
import { localConnectionSchema } from "@solid-imager/core/domain/sources/schemas";
import { getErrorMessage } from "@solid-imager/core/utils/get-error-message";
import type { Table } from "drizzle-orm";
-import { and, asc, eq, gt, inArray, lt, sql } from "drizzle-orm";
+import { and, asc, eq, gt, inArray, lt, or, sql } from "drizzle-orm";
import type { PgColumn } from "drizzle-orm/pg-core";
import { LANCEDB_DUMP_VERSION } from "~/application/services/lancedb-dump-service";
import { db } from "~/infrastructure/db";
@@ -260,10 +261,38 @@ export const BackupService = {
const connectionInfo = mediaSource.connectionInfo as { path: string };
const basePath = connectionInfo.path;
+ const revisionRows = await db
+ .select({
+ id: medias.id,
+ mediaSourceId: medias.mediaSourceId,
+ modifiedAt: medias.modifiedAt,
+ fileSize: medias.fileSize,
+ width: medias.width,
+ height: medias.height,
+ })
+ .from(medias)
+ .where(inArray(medias.id, mediaIds));
+ const revisionById = new Map(
+ await Promise.all(
+ revisionRows.map(async (media) => [
+ media.id,
+ await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ }),
+ ] as const),
+ ),
+ );
for (const id of mediaIds) {
await jobRepo.create({
type: "processMedia",
mediaSourceId,
+ targetId: id,
+ inputRevision: revisionById.get(id) ?? null,
payload: {
mediaId: id,
sourcePath: basePath,
@@ -1079,6 +1108,8 @@ export const BackupService = {
target: [lanceDbSyncDirty.mediaSourceId, lanceDbSyncDirty.mediaId],
set: {
operation,
+ generation: sql`${lanceDbSyncDirty.generation} + 1`,
+ attempts: 0,
lastError: null,
updatedAt: now,
},
@@ -1199,12 +1230,17 @@ export const BackupService = {
itemsToUpsert: upsertItems,
});
- await db.delete(lanceDbSyncDirty).where(
- inArray(
- lanceDbSyncDirty.id,
- dirtyRows.map((row) => row.id),
+ const claimedGenerations = or(
+ ...dirtyRows.map((row) =>
+ and(
+ eq(lanceDbSyncDirty.id, row.id),
+ eq(lanceDbSyncDirty.generation, row.generation),
+ ),
),
);
+ if (claimedGenerations) {
+ await db.delete(lanceDbSyncDirty).where(claimedGenerations);
+ }
logger.info(
{
@@ -1219,19 +1255,24 @@ export const BackupService = {
return { mode: "delta", processed: dirtyRows.length };
} catch (error) {
const message = getErrorMessage(error);
- await db
- .update(lanceDbSyncDirty)
- .set({
- attempts: sql`${lanceDbSyncDirty.attempts} + 1`,
- lastError: message,
- updatedAt: new Date(),
- })
- .where(
- inArray(
- lanceDbSyncDirty.id,
- dirtyRows.map((row) => row.id),
+ const claimedGenerations = or(
+ ...dirtyRows.map((row) =>
+ and(
+ eq(lanceDbSyncDirty.id, row.id),
+ eq(lanceDbSyncDirty.generation, row.generation),
),
- );
+ ),
+ );
+ if (claimedGenerations) {
+ await db
+ .update(lanceDbSyncDirty)
+ .set({
+ attempts: sql`${lanceDbSyncDirty.attempts} + 1`,
+ lastError: message,
+ updatedAt: new Date(),
+ })
+ .where(claimedGenerations);
+ }
throw error;
}
},
diff --git a/apps/server/src/application/services/ccip-vector-service.ts b/apps/server/src/application/services/ccip-vector-service.ts
index c4ba568ba..580ebd757 100644
--- a/apps/server/src/application/services/ccip-vector-service.ts
+++ b/apps/server/src/application/services/ccip-vector-service.ts
@@ -3,6 +3,8 @@ import { CcipVectorService } from "@solid-imager/application/services/ccip-vecto
import { services } from "~/application/registry";
import { taggingService } from "~/application/services/tagging-service";
import { PostgresCcipVectorStore } from "~/infrastructure/ai/postgres-ccip-vector-store";
+import { LanceDbCcipVectorStore } from "~/infrastructure/ai/lancedb-ccip-vector-store";
+import { DualWriteCcipVectorStore } from "~/infrastructure/ai/dual-write-ccip-vector-store";
import { db } from "~/infrastructure/db";
let service: CcipVectorService | null = null;
@@ -14,11 +16,49 @@ export function configureCcipVectorService(logger: ILogger): void {
export function getCcipVectorService(): CcipVectorService {
if (!service) {
+ const config = services.getConfigService().getConfig();
+ const postgresStore = new PostgresCcipVectorStore(db, configuredLogger);
+ const legacyLanceStore = new LanceDbCcipVectorStore(
+ config.lancedb.ccipVectorDir,
+ { legacy: true },
+ );
+ const rollbackStore = new LanceDbCcipVectorStore(
+ config.lancedb.ccipRollbackDir,
+ { readOnly: config.lancedb.ccipStoreMode === "lance-readonly" },
+ );
+ const vectorStore = (() => {
+ switch (config.lancedb.ccipStoreMode) {
+ case "lance":
+ return legacyLanceStore;
+ case "postgres":
+ return postgresStore;
+ case "postgres-dual-write":
+ return new DualWriteCcipVectorStore(
+ postgresStore,
+ [
+ { name: "postgres", store: postgresStore },
+ { name: "lance-rollback", store: rollbackStore },
+ ],
+ configuredLogger,
+ );
+ case "lance-dual-write":
+ return new DualWriteCcipVectorStore(
+ rollbackStore,
+ [
+ { name: "lance-rollback", store: rollbackStore },
+ { name: "postgres", store: postgresStore },
+ ],
+ configuredLogger,
+ );
+ case "lance-readonly":
+ return rollbackStore;
+ }
+ })();
service = new CcipVectorService({
mediaRepository: services.getMediaRepository(),
sourceRepository: services.getSourceRepository(),
taggingService,
- vectorStore: new PostgresCcipVectorStore(db, configuredLogger),
+ vectorStore,
logger: configuredLogger,
});
}
diff --git a/apps/server/src/application/services/job-dispatch-service.ts b/apps/server/src/application/services/job-dispatch-service.ts
index bc4c9882b..658c09c5a 100644
--- a/apps/server/src/application/services/job-dispatch-service.ts
+++ b/apps/server/src/application/services/job-dispatch-service.ts
@@ -1,18 +1,42 @@
import type { DeferredActions } from "@solid-imager/application/ports/media-service";
+import { validateJobPayload } from "@solid-imager/core/domain/jobs/registry";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
+import type { Job } from "@solid-imager/core/domain/repositories/job-repository";
+import { eq } from "drizzle-orm";
import { services } from "~/application/registry";
-import type { Job as DbJob } from "~/infrastructure/db/schema";
+import { db } from "~/infrastructure/db";
+import { medias } from "~/infrastructure/db/schema";
import { RealtimeEventBus } from "~/infrastructure/events/realtime-event-bus";
import {
processAutoTaggingJob,
processBulkTaggingDispatchJob,
} from "~/infrastructure/jobs/tagging-jobs";
import { deleteThumbnail } from "~/infrastructure/jobs/thumbnails";
+import { NonRetryableJobError } from "~/infrastructure/jobs/job-errors";
import { logger } from "~/infrastructure/logger";
// Helper for unified job processing (Called by JobWorker)
-export async function processJob(job: DbJob) {
+export async function processJob(job: Job, signal?: AbortSignal) {
+ const validated = validateJobPayload(job.type, job.payload);
+ if (!validated.success) {
+ const issueSummary = validated.error.issues
+ .map((issue) => `${issue.path.join(".")}: ${issue.message}`)
+ .join("; ");
+ throw new NonRetryableJobError(
+ validated.error.issues.some((issue) => issue.path[0] === "type")
+ ? "UNKNOWN_JOB_TYPE"
+ : "INVALID_JOB_PAYLOAD",
+ `Invalid ${job.type} job: ${issueSummary}`,
+ );
+ }
+ if (signal?.aborted) return;
+ await assertCurrentInputRevision(job);
const mediaSourceId = job.mediaSourceId;
- if (!mediaSourceId && job.type !== "bulk_tagging_dispatch") {
+ if (
+ !mediaSourceId &&
+ job.type !== "bulk_tagging_dispatch" &&
+ job.type !== "batch_ccip_dispatch"
+ ) {
throw new Error(`Job ${job.id} missing mediaSourceId`);
}
@@ -27,12 +51,12 @@ export async function processJob(job: DbJob) {
);
await processDownloadJob(job);
} else if (job.type === "auto_tagging") {
- await processAutoTaggingJob(job);
+ await processAutoTaggingJob(job, signal);
} else if (job.type === "extract_ccip_vector") {
const { processCcipExtractionJob } = await import(
"~/infrastructure/jobs/ccip-jobs"
);
- await processCcipExtractionJob(job);
+ await processCcipExtractionJob(job, signal);
} else if (job.type === "bulk_tagging_dispatch") {
await processBulkTaggingDispatchJob(job);
} else if (job.type === "batch_ccip_dispatch") {
@@ -56,19 +80,64 @@ export async function processJob(job: DbJob) {
"~/application/services/backup-service"
);
const batchSize = getDeltaBatchSize(job.payload);
- const payloadDirty = getDeltaDirtyPayload(job.payload);
- if (payloadDirty.mediaIds.length > 0) {
- await BackupService.queueSourceLanceDBDelta(
- mediaSourceId,
- payloadDirty.mediaIds,
- payloadDirty.operation,
- { enqueueJob: false },
- );
- }
await BackupService.syncSourceLanceDBDeltaCache(mediaSourceId, batchSize);
} else {
- logger.warn({ jobId: job.id, type: job.type }, "Unknown job type");
+ throw new NonRetryableJobError(
+ "UNKNOWN_JOB_TYPE",
+ `Unknown job type: ${job.type}`,
+ );
}
+ if (!signal?.aborted) await assertCurrentInputRevision(job);
+}
+
+async function assertCurrentInputRevision(job: Job): Promise {
+ if (
+ !job.inputRevision ||
+ !["processMedia", "auto_tagging", "extract_ccip_vector"].includes(job.type)
+ ) {
+ return;
+ }
+ const mediaId = job.targetId ?? getPayloadMediaId(job.payload);
+ if (!mediaId) return;
+ const [media] = await db
+ .select({
+ id: medias.id,
+ mediaSourceId: medias.mediaSourceId,
+ modifiedAt: medias.modifiedAt,
+ fileSize: medias.fileSize,
+ width: medias.width,
+ height: medias.height,
+ })
+ .from(medias)
+ .where(eq(medias.id, mediaId))
+ .limit(1);
+ if (!media) {
+ throw new NonRetryableJobError(
+ "TARGET_NOT_FOUND",
+ `Job target media not found: ${mediaId}`,
+ );
+ }
+ const currentRevision = await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
+ if (currentRevision !== job.inputRevision) {
+ throw new NonRetryableJobError(
+ "STALE_INPUT",
+ `Job input revision is stale for media ${mediaId}`,
+ );
+ }
+}
+
+function getPayloadMediaId(payload: unknown): string | null {
+ if (!payload || typeof payload !== "object" || !("mediaId" in payload)) {
+ return null;
+ }
+ return typeof payload.mediaId === "string" ? payload.mediaId : null;
}
function getDeltaBatchSize(payload: unknown): number {
@@ -83,29 +152,6 @@ function getDeltaBatchSize(payload: unknown): number {
return 500;
}
-function getDeltaDirtyPayload(payload: unknown): {
- mediaIds: string[];
- operation: "upsert" | "delete";
-} {
- if (!payload || typeof payload !== "object") {
- return { mediaIds: [], operation: "upsert" };
- }
- const data = payload as {
- mediaIds?: unknown;
- mediaId?: unknown;
- operation?: unknown;
- };
- const mediaIds = Array.isArray(data.mediaIds)
- ? data.mediaIds.filter(
- (value): value is string => typeof value === "string",
- )
- : typeof data.mediaId === "string"
- ? [data.mediaId]
- : [];
- const operation = data.operation === "delete" ? "delete" : "upsert";
- return { mediaIds, operation };
-}
-
export async function executeDeferredActions(actions: DeferredActions) {
if (actions.jobs.length > 0) {
const repo = services.getJobRepository();
@@ -129,6 +175,8 @@ export async function executeDeferredActions(actions: DeferredActions) {
await repo.create({
type: job.type,
mediaSourceId: item.mediaSourceId,
+ targetId: job.targetId,
+ inputRevision: job.inputRevision,
payload: jobPayload,
});
}
diff --git a/apps/server/src/application/services/maintenance-service.ts b/apps/server/src/application/services/maintenance-service.ts
index c517a9783..124496f20 100644
--- a/apps/server/src/application/services/maintenance-service.ts
+++ b/apps/server/src/application/services/maintenance-service.ts
@@ -1,5 +1,6 @@
import fs from "node:fs/promises";
import path from "node:path";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
import type { IMediaRepository } from "@solid-imager/core/domain/repositories/media-repository";
import type { SourceRepository } from "@solid-imager/core/domain/repositories/source-repository";
import type { IJobRepository } from "~/domain/repositories/job-repository";
@@ -308,9 +309,22 @@ export class MaintenanceService {
}
try {
+ const media = await this.mediaRepo.findById(item.id);
+ const inputRevision = media
+ ? await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ })
+ : null;
return await this.jobRepo.createIfUnique({
type: "processMedia",
mediaSourceId: item.mediaSourceId,
+ targetId: item.id,
+ inputRevision,
payload: {
mediaId: item.id,
sourcePath: basePath,
diff --git a/apps/server/src/components/media/character-crop-modal.tsx b/apps/server/src/components/media/character-crop-modal.tsx
index cb476a406..a6f55ff9b 100644
--- a/apps/server/src/components/media/character-crop-modal.tsx
+++ b/apps/server/src/components/media/character-crop-modal.tsx
@@ -1,6 +1,14 @@
import type { MediaDetails } from "@solid-imager/core/domain/media/schemas";
import { CharacterCropModal as SharedCharacterCropModal } from "@solid-imager/ui/character-crop-modal";
-import { fetchCharacterCrops } from "~/infrastructure/api-clients/ai-api";
+import {
+ createManualMediaRegion,
+ deleteMediaRegion,
+ fetchCharacterCrops,
+ fetchMediaRegions,
+ getMediaRegionRenderUrl,
+ materializeMediaRegion,
+ updateMediaRegion,
+} from "~/infrastructure/api-clients/ai-api";
type CharacterCropModalProps = {
isOpen: boolean;
@@ -11,12 +19,22 @@ type CharacterCropModalProps = {
export default function CharacterCropModal(props: CharacterCropModalProps) {
return (
{
- return fetchCharacterCrops(mediaId, transparent);
+ createManualRegion={createManualMediaRegion}
+ deleteRegion={deleteMediaRegion}
+ detectRegions={async (mediaId: string) => {
+ const result = await fetchCharacterCrops(mediaId, false);
+ if (result.mode !== "media-backed") {
+ throw new Error("Character detection did not return saved regions.");
+ }
+ return result.regions;
}}
+ getRenderUrl={getMediaRegionRenderUrl}
isOpen={props.isOpen}
+ loadRegions={fetchMediaRegions}
+ materializeRegion={materializeMediaRegion}
media={props.media}
onClose={props.onClose}
+ updateRegion={updateMediaRegion}
/>
);
}
diff --git a/apps/server/src/domain/shared/api-contract.ts b/apps/server/src/domain/shared/api-contract.ts
index 571dc826b..cc8d886c0 100644
--- a/apps/server/src/domain/shared/api-contract.ts
+++ b/apps/server/src/domain/shared/api-contract.ts
@@ -8,6 +8,7 @@ import { downloadsRouter } from "~/infrastructure/api/routers/downloads-router";
import { importsRouter } from "~/infrastructure/api/routers/imports-router";
import { ipsRouter } from "~/infrastructure/api/routers/ips-router";
import { jobsRouter } from "~/infrastructure/api/routers/jobs-router";
+import { mediaRegionsRouter } from "~/infrastructure/api/routers/media-regions-router";
import { mediaRouter } from "~/infrastructure/api/routers/media-router";
import { presetsRouter } from "~/infrastructure/api/routers/presets-router";
import { projectsRouter } from "~/infrastructure/api/routers/projects-router";
@@ -24,6 +25,7 @@ export const appRouter = {
sources: sourcesRouter,
tags: tagsRouter,
media: mediaRouter,
+ mediaRegions: mediaRegionsRouter,
categories: categoriesRouter,
projects: projectsRouter,
characters: charactersRouter,
diff --git a/apps/server/src/infrastructure/ai/dual-write-ccip-vector-store.ts b/apps/server/src/infrastructure/ai/dual-write-ccip-vector-store.ts
new file mode 100644
index 000000000..00dd66555
--- /dev/null
+++ b/apps/server/src/infrastructure/ai/dual-write-ccip-vector-store.ts
@@ -0,0 +1,152 @@
+import type {
+ CcipVectorCandidate,
+ CcipEmbeddingKey,
+ CcipVectorMetadata,
+ CcipVectorQuery,
+ CcipVectorReadQuery,
+ CcipVectorRecord,
+ ICcipVectorStore,
+} from "@solid-imager/application/ports/ccip-vector-store";
+import type { ILogger } from "@solid-imager/application/ports/media-service";
+
+export type CcipWriteBackend = {
+ name: string;
+ store: ICcipVectorStore;
+};
+
+export class CcipDualWriteError extends Error {
+ constructor(
+ readonly operation: string,
+ readonly succeededBackends: string[],
+ readonly failedBackend: string,
+ cause: unknown,
+ ) {
+ super(
+ `CCIP ${operation} partially failed at ${failedBackend} after ${succeededBackends.join(", ") || "no successful backend"}`,
+ { cause },
+ );
+ this.name = "CcipDualWriteError";
+ }
+}
+
+/**
+ * Transitional store used only during the rollback observation window. Reads
+ * stay on one authoritative backend while every mutation is synchronously
+ * applied to both backends; a secondary failure is never hidden.
+ */
+export class DualWriteCcipVectorStore implements ICcipVectorStore {
+ constructor(
+ private readonly readStore: ICcipVectorStore,
+ private readonly writeBackends: CcipWriteBackend[],
+ private readonly logger?: ILogger,
+ ) {}
+
+ private async write(
+ operation: string,
+ callback: (store: ICcipVectorStore) => Promise,
+ ): Promise {
+ const succeededBackends: string[] = [];
+ for (const backend of this.writeBackends) {
+ try {
+ await callback(backend.store);
+ succeededBackends.push(backend.name);
+ } catch (error) {
+ this.logger?.error(
+ {
+ err: error,
+ operation,
+ failedBackend: backend.name,
+ succeededBackends,
+ },
+ "CCIP dual-write operation failed",
+ );
+ throw new CcipDualWriteError(
+ operation,
+ [...succeededBackends],
+ backend.name,
+ error,
+ );
+ }
+ }
+ }
+
+ async get(
+ mediaId: string,
+ query: CcipVectorReadQuery,
+ ): Promise {
+ return await this.readStore.get(mediaId, query);
+ }
+
+ async getByRegion(
+ regionId: string,
+ query: CcipVectorReadQuery,
+ ): Promise {
+ return await this.readStore.getByRegion(regionId, query);
+ }
+
+ async getMany(
+ mediaIds: string[],
+ query: CcipVectorReadQuery,
+ ): Promise> {
+ return await this.readStore.getMany(mediaIds, query);
+ }
+
+ async getMetadataMany(
+ mediaIds: string[],
+ query: CcipVectorReadQuery,
+ ): Promise> {
+ return await this.readStore.getMetadataMany(mediaIds, query);
+ }
+
+ async upsert(record: CcipVectorRecord): Promise {
+ await this.write("upsert", async (store) => await store.upsert(record));
+ }
+
+ async upsertMany(records: CcipVectorRecord[]): Promise {
+ await this.write(
+ "upsertMany",
+ async (store) => await store.upsertMany(records),
+ );
+ }
+
+ async delete(mediaId: string): Promise {
+ await this.write("delete", async (store) => await store.delete(mediaId));
+ }
+
+ async deleteRegion(regionId: string): Promise {
+ await this.write(
+ "deleteRegion",
+ async (store) => await store.deleteRegion(regionId),
+ );
+ }
+
+ async deleteEmbedding(key: CcipEmbeddingKey): Promise {
+ await this.write(
+ "deleteEmbedding",
+ async (store) => await store.deleteEmbedding(key),
+ );
+ }
+
+ async deleteBySource(mediaSourceId: string): Promise {
+ await this.write(
+ "deleteBySource",
+ async (store) => await store.deleteBySource(mediaSourceId),
+ );
+ }
+
+ async listMediaIds(query?: CcipVectorQuery): Promise {
+ return await this.readStore.listMediaIds(query);
+ }
+
+ async list(query?: CcipVectorQuery): Promise {
+ return await this.readStore.list(query);
+ }
+
+ async search(
+ vector: number[],
+ limit: number,
+ query: CcipVectorReadQuery,
+ ): Promise {
+ return await this.readStore.search(vector, limit, query);
+ }
+}
diff --git a/apps/server/src/infrastructure/ai/lancedb-ccip-vector-store.ts b/apps/server/src/infrastructure/ai/lancedb-ccip-vector-store.ts
index 8710dcfe2..4b01bc620 100644
--- a/apps/server/src/infrastructure/ai/lancedb-ccip-vector-store.ts
+++ b/apps/server/src/infrastructure/ai/lancedb-ccip-vector-store.ts
@@ -2,6 +2,7 @@ import * as path from "node:path";
import type { Connection, Table } from "@lancedb/lancedb";
import type {
CcipVectorCandidate,
+ CcipEmbeddingKey,
CcipVectorMetadata,
CcipVectorQuery,
CcipVectorReadQuery,
@@ -13,8 +14,28 @@ import { z } from "zod";
const TABLE_NAME = "media_ccip";
const VECTOR_DIMENSIONS = 768;
const AUTO_OPTIMIZE_WRITE_OPERATIONS = 100;
+const LEGACY_INPUT_REVISION = "legacy-unversioned";
+const DEFAULT_PREPROCESSING_PROFILE =
+ "dghs-imgutils-rs/full-image-default/v1";
+
+export class CcipStoreReadOnlyError extends Error {
+ constructor() {
+ super("CCIP LanceDB store is read-only");
+ this.name = "CcipStoreReadOnlyError";
+ }
+}
+
+export class CcipStoreUnsupportedOperationError extends Error {
+ constructor(readonly operation: string) {
+ super(`${operation} is unavailable in legacy LanceDB mode`);
+ this.name = "CcipStoreUnsupportedOperationError";
+ }
+}
const rowSchema = z.object({
+ regionKey: z.string().optional(),
+ regionId: z.string().uuid().nullable().optional(),
+ regionKind: z.enum(["full", "person", "manual"]).optional(),
mediaId: z.string().uuid(),
mediaSourceId: z.string().uuid(),
vector: z.preprocess((value) => {
@@ -32,6 +53,8 @@ const rowSchema = z.object({
model: z.string(),
embeddingVersion: z.number().int(),
mediaModifiedAt: z.coerce.date(),
+ inputRevision: z.string().min(1).optional(),
+ preprocessingProfile: z.string().min(1).optional(),
extractedAt: z.coerce.date(),
_distance: z.number().optional(),
});
@@ -45,8 +68,17 @@ function escapeSqlString(value: string): string {
return value.replaceAll("'", "''");
}
-function queryPredicates(query?: CcipVectorQuery): string[] {
+function queryPredicates(
+ query?: CcipVectorQuery,
+ includeVersionedColumns = true,
+): string[] {
const predicates: string[] = [];
+ if (includeVersionedColumns && query?.regionId) {
+ predicates.push(`regionId = '${escapeSqlString(query.regionId)}'`);
+ }
+ if (includeVersionedColumns && query?.regionKind) {
+ predicates.push(`regionKind = '${escapeSqlString(query.regionKind)}'`);
+ }
if (query?.mediaSourceId) {
predicates.push(
`mediaSourceId = '${escapeSqlString(query.mediaSourceId)}'`,
@@ -58,24 +90,47 @@ function queryPredicates(query?: CcipVectorQuery): string[] {
if (query?.embeddingVersion !== undefined) {
predicates.push(`embeddingVersion = ${query.embeddingVersion}`);
}
+ if (includeVersionedColumns && query?.preprocessingProfile) {
+ predicates.push(
+ `preprocessingProfile = '${escapeSqlString(query.preprocessingProfile)}'`,
+ );
+ }
return predicates;
}
function toRecord(value: unknown): CcipVectorRecord {
const row = rowSchema.parse(value);
return {
+ regionId: row.regionId ?? null,
+ regionKind: row.regionKind ?? "full",
mediaId: row.mediaId,
mediaSourceId: row.mediaSourceId,
vector: row.vector,
model: row.model,
embeddingVersion: row.embeddingVersion,
mediaModifiedAt: row.mediaModifiedAt,
+ inputRevision: row.inputRevision ?? LEGACY_INPUT_REVISION,
+ preprocessingProfile:
+ row.preprocessingProfile ?? DEFAULT_PREPROCESSING_PROFILE,
extractedAt: row.extractedAt,
};
}
function toMetadata(value: unknown): CcipVectorMetadata {
- return metadataRowSchema.parse(value);
+ const row = metadataRowSchema.parse(value);
+ return {
+ regionId: row.regionId ?? null,
+ regionKind: row.regionKind ?? "full",
+ mediaId: row.mediaId,
+ mediaSourceId: row.mediaSourceId,
+ model: row.model,
+ embeddingVersion: row.embeddingVersion,
+ mediaModifiedAt: row.mediaModifiedAt,
+ inputRevision: row.inputRevision ?? LEGACY_INPUT_REVISION,
+ preprocessingProfile:
+ row.preprocessingProfile ?? DEFAULT_PREPROCESSING_PROFILE,
+ extractedAt: row.extractedAt,
+ };
}
export class LanceDbCcipVectorStore implements ICcipVectorStore {
@@ -86,7 +141,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
constructor(
private readonly directory: string,
- private readonly options: { readOnly?: boolean } = {},
+ private readonly options: { readOnly?: boolean; legacy?: boolean } = {},
) {}
private async connection(): Promise {
@@ -133,7 +188,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
);
}
const arrow = await import("apache-arrow");
- const schema = new arrow.Schema([
+ const sharedFields = [
new arrow.Field("mediaId", new arrow.Utf8(), false),
new arrow.Field("mediaSourceId", new arrow.Utf8(), false),
new arrow.Field(
@@ -152,7 +207,23 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
false,
),
new arrow.Field("extractedAt", new arrow.TimestampMillisecond(), false),
- ]);
+ ];
+ const schema = new arrow.Schema(
+ this.options.legacy
+ ? sharedFields
+ : [
+ new arrow.Field("regionKey", new arrow.Utf8(), false),
+ new arrow.Field("regionId", new arrow.Utf8(), true),
+ new arrow.Field("regionKind", new arrow.Utf8(), false),
+ ...sharedFields,
+ new arrow.Field("inputRevision", new arrow.Utf8(), false),
+ new arrow.Field(
+ "preprocessingProfile",
+ new arrow.Utf8(),
+ false,
+ ),
+ ],
+ );
table = await db.createTable(TABLE_NAME, [], { schema });
}
if (!this.options.readOnly) {
@@ -175,7 +246,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
private async serializeWrite(operation: () => Promise): Promise {
if (this.options.readOnly) {
- throw new Error("CCIP LanceDB store is read-only");
+ throw new CcipStoreReadOnlyError();
}
const next = this.writeQueue.then(operation, operation);
this.writeQueue = next.catch(() => undefined);
@@ -189,6 +260,25 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
return (await this.getMany([mediaId], query)).get(mediaId) ?? null;
}
+ async getByRegion(
+ regionId: string,
+ query: CcipVectorReadQuery,
+ ): Promise {
+ if (this.options.legacy) return null;
+ const table = await this.table();
+ const rows = await table
+ .query()
+ .where(
+ [
+ `regionId = '${escapeSqlString(regionId)}'`,
+ ...queryPredicates(query, true),
+ ].join(" AND "),
+ )
+ .limit(1)
+ .toArray();
+ return rows[0] ? toRecord(rows[0]) : null;
+ }
+
async getMany(
mediaIds: string[],
query: CcipVectorReadQuery,
@@ -198,7 +288,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
}
const predicates = [
`mediaId IN (${mediaIds.map((mediaId) => `'${escapeSqlString(mediaId)}'`).join(", ")})`,
- ...queryPredicates(query),
+ ...queryPredicates(query, !this.options.legacy),
];
const table = await this.table();
const rows = await table.query().where(predicates.join(" AND ")).toArray();
@@ -219,19 +309,28 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
}
const predicates = [
`mediaId IN (${mediaIds.map((mediaId) => `'${escapeSqlString(mediaId)}'`).join(", ")})`,
- ...queryPredicates(query),
+ ...queryPredicates(query, !this.options.legacy),
];
const table = await this.table();
- const rows = await table
- .query()
- .select([
+ const selectedColumns = [
"mediaId",
"mediaSourceId",
"model",
"embeddingVersion",
"mediaModifiedAt",
"extractedAt",
- ])
+ ];
+ if (!this.options.legacy) {
+ selectedColumns.push(
+ "regionId",
+ "regionKind",
+ "inputRevision",
+ "preprocessingProfile",
+ );
+ }
+ const rows = await table
+ .query()
+ .select(selectedColumns)
.where(predicates.join(" AND "))
.toArray();
return new Map(
@@ -252,11 +351,34 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
}
await this.serializeWrite(async () => {
const table = await this.table();
+ if (this.options.legacy) {
+ await table
+ .mergeInsert("mediaId")
+ .whenMatchedUpdateAll()
+ .whenNotMatchedInsertAll()
+ .execute(
+ records.map((record) => ({
+ mediaId: record.mediaId,
+ mediaSourceId: record.mediaSourceId,
+ vector: record.vector,
+ model: record.model,
+ embeddingVersion: record.embeddingVersion,
+ mediaModifiedAt: record.mediaModifiedAt,
+ extractedAt: record.extractedAt,
+ })),
+ );
+ return;
+ }
await table
- .mergeInsert("mediaId")
+ .mergeInsert("regionKey")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
- .execute(records);
+ .execute(
+ records.map((record) => ({
+ ...record,
+ regionKey: record.regionId ?? `full:${record.mediaId}`,
+ })),
+ );
this.writeOperationsSinceOptimize++;
if (this.writeOperationsSinceOptimize >= AUTO_OPTIMIZE_WRITE_OPERATIONS) {
await table.optimize();
@@ -273,6 +395,33 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
});
}
+ async deleteRegion(regionId: string): Promise {
+ if (this.options.legacy) {
+ throw new CcipStoreUnsupportedOperationError("deleteRegion");
+ }
+ await this.serializeWrite(async () => {
+ const table = await this.table();
+ await table.delete(`regionId = '${escapeSqlString(regionId)}'`);
+ });
+ }
+
+ async deleteEmbedding(key: CcipEmbeddingKey): Promise {
+ if (this.options.legacy) {
+ throw new CcipStoreUnsupportedOperationError("deleteEmbedding");
+ }
+ await this.serializeWrite(async () => {
+ const table = await this.table();
+ await table.delete(
+ [
+ `regionId = '${escapeSqlString(key.regionId)}'`,
+ `model = '${escapeSqlString(key.model)}'`,
+ `embeddingVersion = ${key.embeddingVersion}`,
+ `preprocessingProfile = '${escapeSqlString(key.preprocessingProfile)}'`,
+ ].join(" AND "),
+ );
+ });
+ }
+
async deleteBySource(mediaSourceId: string): Promise {
await this.serializeWrite(async () => {
const table = await this.table();
@@ -283,7 +432,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
async listMediaIds(query?: CcipVectorQuery): Promise {
const table = await this.table();
const tableQuery = table.query().select(["mediaId"]);
- const predicates = queryPredicates(query);
+ const predicates = queryPredicates(query, !this.options.legacy);
if (predicates.length > 0) {
tableQuery.where(predicates.join(" AND "));
}
@@ -297,7 +446,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
async list(query?: CcipVectorQuery): Promise {
const table = await this.table();
const tableQuery = table.query();
- const predicates = queryPredicates(query);
+ const predicates = queryPredicates(query, !this.options.legacy);
if (predicates.length > 0) {
tableQuery.where(predicates.join(" AND "));
}
@@ -318,7 +467,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
throw new Error("batchSize must be a positive integer");
}
const table = await this.table();
- const predicates = queryPredicates(query);
+ const predicates = queryPredicates(query, !this.options.legacy);
let offset = 0;
while (true) {
const tableQuery = table
@@ -356,7 +505,7 @@ export class LanceDbCcipVectorStore implements ICcipVectorStore {
.vectorSearch(vector)
.distanceType("cosine")
.limit(limit);
- const predicates = queryPredicates(query);
+ const predicates = queryPredicates(query, !this.options.legacy);
if (predicates.length > 0) {
tableQuery.where(predicates.join(" AND "));
}
diff --git a/apps/server/src/infrastructure/ai/postgres-ccip-vector-store.ts b/apps/server/src/infrastructure/ai/postgres-ccip-vector-store.ts
index e43472ef8..1e2a1efd9 100644
--- a/apps/server/src/infrastructure/ai/postgres-ccip-vector-store.ts
+++ b/apps/server/src/infrastructure/ai/postgres-ccip-vector-store.ts
@@ -1,5 +1,6 @@
import type {
CcipVectorCandidate,
+ CcipEmbeddingKey,
CcipVectorMetadata,
CcipVectorQuery,
CcipVectorReadQuery,
@@ -7,6 +8,11 @@ import type {
ICcipVectorStore,
} from "@solid-imager/application/ports/ccip-vector-store";
import type { ILogger } from "@solid-imager/application/ports/media-service";
+import {
+ createCcipEmbeddingInputRevision,
+ createMediaRegionRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
import {
CCIP_VECTOR_DIMENSIONS,
ccipEmbeddings,
@@ -20,12 +26,16 @@ import { z } from "zod";
const FULL_REGION_KIND = "full";
const recordRowSchema = z.object({
+ regionId: z.string().uuid(),
+ regionKind: z.enum(["full", "person", "manual"]),
mediaId: z.string().uuid(),
mediaSourceId: z.string().uuid(),
vector: z.array(z.number().finite()).length(CCIP_VECTOR_DIMENSIONS),
model: z.string(),
embeddingVersion: z.number().int(),
mediaModifiedAt: z.coerce.date(),
+ inputRevision: z.string().min(1),
+ preprocessingProfile: z.string().min(1),
extractedAt: z.coerce.date(),
});
@@ -85,7 +95,10 @@ function mapMetadata(value: unknown): CcipVectorMetadata {
function recordFilters(query?: CcipVectorQuery): SQL | undefined {
return and(
- eq(mediaRegions.kind, FULL_REGION_KIND),
+ query?.regionId ? eq(mediaRegions.id, query.regionId) : undefined,
+ query?.regionKind
+ ? eq(mediaRegions.kind, query.regionKind)
+ : eq(mediaRegions.kind, FULL_REGION_KIND),
query?.mediaSourceId
? eq(medias.mediaSourceId, query.mediaSourceId)
: undefined,
@@ -93,19 +106,38 @@ function recordFilters(query?: CcipVectorQuery): SQL | undefined {
query?.embeddingVersion !== undefined
? eq(ccipEmbeddings.embeddingVersion, query.embeddingVersion)
: undefined,
+ query?.preprocessingProfile
+ ? eq(
+ ccipEmbeddings.preprocessingProfile,
+ query.preprocessingProfile,
+ )
+ : undefined,
);
}
const recordColumns = {
+ regionId: mediaRegions.id,
+ regionKind: mediaRegions.kind,
mediaId: medias.id,
mediaSourceId: medias.mediaSourceId,
vector: ccipEmbeddings.embedding,
model: ccipEmbeddings.model,
embeddingVersion: ccipEmbeddings.embeddingVersion,
mediaModifiedAt: ccipEmbeddings.mediaModifiedAt,
+ inputRevision: ccipEmbeddings.inputRevision,
+ preprocessingProfile: ccipEmbeddings.preprocessingProfile,
extractedAt: ccipEmbeddings.extractedAt,
};
+type MediaRevisionRow = {
+ id: string;
+ mediaSourceId: string;
+ modifiedAt: Date;
+ fileSize: number | null;
+ width: number;
+ height: number;
+};
+
/** pgvector-backed CCIP store used by the application at runtime. */
export class PostgresCcipVectorStore implements ICcipVectorStore {
constructor(
@@ -113,6 +145,20 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
private readonly logger?: ILogger,
) {}
+ async getByRegion(
+ regionId: string,
+ query: CcipVectorReadQuery,
+ ): Promise {
+ const rows = await this.database
+ .select(recordColumns)
+ .from(ccipEmbeddings)
+ .innerJoin(mediaRegions, eq(ccipEmbeddings.regionId, mediaRegions.id))
+ .innerJoin(medias, eq(mediaRegions.mediaId, medias.id))
+ .where(recordFilters({ ...query, regionId }))
+ .limit(1);
+ return rows[0] ? mapRecord(rows[0]) : null;
+ }
+
async get(
mediaId: string,
query: CcipVectorReadQuery,
@@ -151,10 +197,14 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
const rows = await this.database
.select({
mediaId: medias.id,
+ regionId: mediaRegions.id,
+ regionKind: mediaRegions.kind,
mediaSourceId: medias.mediaSourceId,
model: ccipEmbeddings.model,
embeddingVersion: ccipEmbeddings.embeddingVersion,
mediaModifiedAt: ccipEmbeddings.mediaModifiedAt,
+ inputRevision: ccipEmbeddings.inputRevision,
+ preprocessingProfile: ccipEmbeddings.preprocessingProfile,
extractedAt: ccipEmbeddings.extractedAt,
})
.from(ccipEmbeddings)
@@ -174,83 +224,242 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
}
async upsertMany(records: CcipVectorRecord[]): Promise {
+ await this.writeMany(records, true);
+ }
+
+ /** One-time migration path. Runtime callers must use revision-fenced upsert. */
+ async importLegacyMany(records: CcipVectorRecord[]): Promise {
+ await this.writeMany(records, false);
+ }
+
+ private async writeMany(
+ records: CcipVectorRecord[],
+ requireCurrentRevision: boolean,
+ ): Promise {
if (records.length === 0) {
return;
}
- const regionsByMediaId = new Map();
- const embeddingsByKey = new Map();
for (const record of records) {
vectorLiteral(record.vector);
- const existingRegion = regionsByMediaId.get(record.mediaId);
- if (
- !existingRegion ||
- record.mediaModifiedAt.getTime() >
- existingRegion.mediaModifiedAt.getTime()
- ) {
- regionsByMediaId.set(record.mediaId, record);
- }
- const embeddingKey = `${record.mediaId}:${record.model}:${record.embeddingVersion}`;
- const existingEmbedding = embeddingsByKey.get(embeddingKey);
- if (
- !existingEmbedding ||
- record.extractedAt.getTime() > existingEmbedding.extractedAt.getTime()
- ) {
- embeddingsByKey.set(embeddingKey, record);
+ if (record.regionKind !== "full" && !record.regionId) {
+ throw new Error("Cropped CCIP embeddings require a regionId");
}
}
const now = new Date();
await this.database.transaction(async (transaction) => {
- const regions = await transaction
- .insert(mediaRegions)
- .values(
- [...regionsByMediaId.values()].map((record) => ({
- mediaId: record.mediaId,
- kind: "full" as const,
- sourceModifiedAt: record.mediaModifiedAt,
- updatedAt: now,
- })),
- )
- .onConflictDoUpdate({
- target: mediaRegions.mediaId,
- targetWhere: sql`${mediaRegions.kind} = 'full'`,
- set: {
- sourceModifiedAt: sql`
- CASE
- WHEN excluded.source_modified_at > ${mediaRegions.sourceModifiedAt}
- THEN excluded.source_modified_at
- ELSE ${mediaRegions.sourceModifiedAt}
- END
- `,
- updatedAt: sql`
- CASE
- WHEN excluded.source_modified_at > ${mediaRegions.sourceModifiedAt}
- THEN excluded.updated_at
- ELSE ${mediaRegions.updatedAt}
- END
- `,
- },
+ const mediaIds = [...new Set(records.map((record) => record.mediaId))];
+ const mediaRows: MediaRevisionRow[] = await transaction
+ .select({
+ id: medias.id,
+ mediaSourceId: medias.mediaSourceId,
+ modifiedAt: medias.modifiedAt,
+ fileSize: medias.fileSize,
+ width: medias.width,
+ height: medias.height,
})
- .returning();
+ .from(medias)
+ .where(inArray(medias.id, mediaIds))
+ .for("share");
+ const mediaById = new Map(mediaRows.map((row) => [row.id, row]));
+ const requestedRegionIds = [
+ ...new Set(
+ records.flatMap((record) =>
+ record.regionId ? [record.regionId] : [],
+ ),
+ ),
+ ];
+ const existingRegionRows =
+ requestedRegionIds.length === 0
+ ? []
+ : await transaction
+ .select({
+ id: mediaRegions.id,
+ mediaId: mediaRegions.mediaId,
+ kind: mediaRegions.kind,
+ sourceRevision: mediaRegions.sourceRevision,
+ })
+ .from(mediaRegions)
+ .where(inArray(mediaRegions.id, requestedRegionIds))
+ .for("share");
+ const existingRegionById = new Map(
+ existingRegionRows.map((region) => [region.id, region]),
+ );
+ const preparedRecords = await Promise.all(
+ records.map(async (record) => {
+ const media = mediaById.get(record.mediaId);
+ if (!media || media.mediaSourceId !== record.mediaSourceId) {
+ throw new Error(
+ `CCIP media is missing or changed source: ${record.mediaId}`,
+ );
+ }
+ const currentSourceRevision = await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
+ const sourceRevision = requireCurrentRevision
+ ? currentSourceRevision
+ : await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: record.mediaModifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
+ const expectedInputRevision =
+ await createCcipEmbeddingInputRevision({
+ sourceRevision,
+ model: record.model,
+ embeddingVersion: record.embeddingVersion,
+ preprocessingProfile: record.preprocessingProfile,
+ });
+ if (
+ requireCurrentRevision &&
+ record.inputRevision !== expectedInputRevision
+ ) {
+ throw new Error(
+ `CCIP input revision changed before commit: ${record.mediaId} (expected ${expectedInputRevision}, received ${record.inputRevision})`,
+ );
+ }
+ if (record.regionId) {
+ const existingRegion = existingRegionById.get(record.regionId);
+ if (
+ existingRegion &&
+ (existingRegion.mediaId !== record.mediaId ||
+ existingRegion.kind !== record.regionKind)
+ ) {
+ throw new Error(`CCIP region identity mismatch: ${record.regionId}`);
+ }
+ if (!existingRegion && record.regionKind !== "full") {
+ throw new Error(`CCIP region does not exist: ${record.regionId}`);
+ }
+ if (
+ requireCurrentRevision &&
+ existingRegion &&
+ existingRegion.sourceRevision !== currentSourceRevision
+ ) {
+ throw new Error(`CCIP region is stale: ${record.regionId}`);
+ }
+ }
+ return {
+ record,
+ media,
+ sourceRevision,
+ inputRevision: expectedInputRevision,
+ };
+ }),
+ );
+ const fullRegionByMediaId = new Map<
+ string,
+ (typeof preparedRecords)[number]
+ >();
+ for (const prepared of preparedRecords) {
+ if (prepared.record.regionKind !== "full") continue;
+ const current = fullRegionByMediaId.get(prepared.record.mediaId);
+ if (
+ !current ||
+ prepared.record.mediaModifiedAt.getTime() >=
+ current.record.mediaModifiedAt.getTime()
+ ) {
+ fullRegionByMediaId.set(prepared.record.mediaId, prepared);
+ }
+ }
+ const regionValues = await Promise.all(
+ [...fullRegionByMediaId.values()].map(async (prepared) => ({
+ id: prepared.record.regionId ?? undefined,
+ mediaId: prepared.record.mediaId,
+ kind: "full" as const,
+ sourceModifiedAt: prepared.record.mediaModifiedAt,
+ sourceWidth: prepared.media.width,
+ sourceHeight: prepared.media.height,
+ sourceRevision: prepared.sourceRevision,
+ regionRevision: await createMediaRegionRevision({
+ sourceRevision: prepared.sourceRevision,
+ kind: "full",
+ x: null,
+ y: null,
+ width: null,
+ height: null,
+ label: null,
+ detector: null,
+ detectorModel: null,
+ detectorVersion: null,
+ manualReason: null,
+ }),
+ updatedAt: now,
+ })),
+ );
+ const regions =
+ regionValues.length === 0
+ ? []
+ : await transaction
+ .insert(mediaRegions)
+ .values(regionValues)
+ .onConflictDoUpdate({
+ target: mediaRegions.mediaId,
+ targetWhere: sql`${mediaRegions.kind} = 'full'`,
+ set: {
+ sourceModifiedAt: sql`CASE WHEN excluded.source_modified_at >= ${mediaRegions.sourceModifiedAt} THEN excluded.source_modified_at ELSE ${mediaRegions.sourceModifiedAt} END`,
+ sourceWidth: sql`CASE WHEN excluded.source_modified_at >= ${mediaRegions.sourceModifiedAt} THEN excluded.source_width ELSE ${mediaRegions.sourceWidth} END`,
+ sourceHeight: sql`CASE WHEN excluded.source_modified_at >= ${mediaRegions.sourceModifiedAt} THEN excluded.source_height ELSE ${mediaRegions.sourceHeight} END`,
+ sourceRevision: sql`CASE WHEN excluded.source_modified_at >= ${mediaRegions.sourceModifiedAt} THEN excluded.source_revision ELSE ${mediaRegions.sourceRevision} END`,
+ regionRevision: sql`CASE WHEN excluded.source_modified_at >= ${mediaRegions.sourceModifiedAt} THEN excluded.region_revision ELSE ${mediaRegions.regionRevision} END`,
+ updatedAt: sql`CASE WHEN excluded.source_modified_at >= ${mediaRegions.sourceModifiedAt} THEN excluded.updated_at ELSE ${mediaRegions.updatedAt} END`,
+ },
+ })
+ .returning();
const regionIdByMediaId = new Map(
regions.map((region) => [region.mediaId, region.id]),
);
- const embeddings = [...embeddingsByKey.values()].map((record) => {
- const regionId = regionIdByMediaId.get(record.mediaId);
+ const embeddingsByKey = new Map<
+ string,
+ {
+ record: CcipVectorRecord;
+ regionId: string;
+ inputRevision: string;
+ }
+ >();
+ for (const prepared of preparedRecords) {
+ const regionId =
+ prepared.record.regionKind === "full"
+ ? regionIdByMediaId.get(prepared.record.mediaId)
+ : prepared.record.regionId;
if (!regionId) {
throw new Error(
- `Unable to create full region for media ${record.mediaId}`,
+ `Unable to resolve CCIP region for media ${prepared.record.mediaId}`,
);
}
- return {
+ const key = `${regionId}:${prepared.record.model}:${prepared.record.embeddingVersion}:${prepared.record.preprocessingProfile}`;
+ const existing = embeddingsByKey.get(key);
+ if (
+ !existing ||
+ prepared.record.extractedAt.getTime() >
+ existing.record.extractedAt.getTime()
+ ) {
+ embeddingsByKey.set(key, {
+ record: prepared.record,
+ regionId,
+ inputRevision: prepared.inputRevision,
+ });
+ }
+ }
+ const embeddings = [...embeddingsByKey.values()].map(
+ ({ record, regionId, inputRevision }) => ({
regionId,
embedding: record.vector,
model: record.model,
embeddingVersion: record.embeddingVersion,
mediaModifiedAt: record.mediaModifiedAt,
+ inputRevision,
+ preprocessingProfile: record.preprocessingProfile,
extractedAt: record.extractedAt,
updatedAt: now,
- };
- });
+ }),
+ );
await transaction
.insert(ccipEmbeddings)
.values(embeddings)
@@ -259,6 +468,7 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
ccipEmbeddings.regionId,
ccipEmbeddings.model,
ccipEmbeddings.embeddingVersion,
+ ccipEmbeddings.preprocessingProfile,
],
set: {
embedding: sql`
@@ -275,6 +485,20 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
ELSE ${ccipEmbeddings.mediaModifiedAt}
END
`,
+ inputRevision: sql`
+ CASE
+ WHEN excluded.extracted_at > ${ccipEmbeddings.extractedAt}
+ THEN excluded.input_revision
+ ELSE ${ccipEmbeddings.inputRevision}
+ END
+ `,
+ preprocessingProfile: sql`
+ CASE
+ WHEN excluded.extracted_at > ${ccipEmbeddings.extractedAt}
+ THEN excluded.preprocessing_profile
+ ELSE ${ccipEmbeddings.preprocessingProfile}
+ END
+ `,
extractedAt: sql`
CASE
WHEN excluded.extracted_at > ${ccipEmbeddings.extractedAt}
@@ -304,6 +528,26 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
.where(inArray(ccipEmbeddings.regionId, regionIds));
}
+ async deleteRegion(regionId: string): Promise {
+ await this.database
+ .delete(ccipEmbeddings)
+ .where(eq(ccipEmbeddings.regionId, regionId));
+ }
+
+ async deleteEmbedding(key: CcipEmbeddingKey): Promise {
+ await this.database.delete(ccipEmbeddings).where(
+ and(
+ eq(ccipEmbeddings.regionId, key.regionId),
+ eq(ccipEmbeddings.model, key.model),
+ eq(ccipEmbeddings.embeddingVersion, key.embeddingVersion),
+ eq(
+ ccipEmbeddings.preprocessingProfile,
+ key.preprocessingProfile,
+ ),
+ ),
+ );
+ }
+
async deleteBySource(mediaSourceId: string): Promise {
const regionIds = this.database
.select({ id: mediaRegions.id })
@@ -345,7 +589,12 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
}
const literal = vectorLiteral(vector);
const filters = [
- sql`${mediaRegions.kind} = ${FULL_REGION_KIND}`,
+ query?.regionId
+ ? sql`${mediaRegions.id} = ${query.regionId}`
+ : undefined,
+ query?.regionKind
+ ? sql`${mediaRegions.kind} = ${query.regionKind}`
+ : sql`${mediaRegions.kind} = ${FULL_REGION_KIND}`,
query?.mediaSourceId
? sql`${medias.mediaSourceId} = ${query.mediaSourceId}`
: undefined,
@@ -353,20 +602,23 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
query?.embeddingVersion !== undefined
? sql`${ccipEmbeddings.embeddingVersion} = ${query.embeddingVersion}`
: undefined,
+ query?.preprocessingProfile
+ ? sql`${ccipEmbeddings.preprocessingProfile} = ${query.preprocessingProfile}`
+ : undefined,
].filter((value): value is SQL => value !== undefined);
const startedAt = performance.now();
- const candidates = await this.database.transaction(async (transaction) => {
- await transaction.execute(
- sql`SET LOCAL hnsw.iterative_scan = 'relaxed_order'`,
- );
- const raw = await transaction.execute(sql`
+ const raw = await this.database.execute(sql`
SELECT
+ ${mediaRegions.id} AS "regionId",
+ ${mediaRegions.kind} AS "regionKind",
${medias.id} AS "mediaId",
${medias.mediaSourceId} AS "mediaSourceId",
${ccipEmbeddings.embedding}::text AS "vector",
${ccipEmbeddings.model} AS "model",
${ccipEmbeddings.embeddingVersion} AS "embeddingVersion",
${ccipEmbeddings.mediaModifiedAt} AS "mediaModifiedAt",
+ ${ccipEmbeddings.inputRevision} AS "inputRevision",
+ ${ccipEmbeddings.preprocessingProfile} AS "preprocessingProfile",
${ccipEmbeddings.extractedAt} AS "extractedAt",
(${ccipEmbeddings.embedding} <=> ${literal}::vector) AS "cosineDistance"
FROM ${ccipEmbeddings}
@@ -375,9 +627,10 @@ export class PostgresCcipVectorStore implements ICcipVectorStore {
WHERE ${sql.join(filters, sql` AND `)}
ORDER BY ${ccipEmbeddings.embedding} <=> ${literal}::vector
LIMIT ${limit}
- `);
- return extractRows(raw).map((row) => rawCandidateRowSchema.parse(row));
- });
+ `);
+ const candidates = extractRows(raw).map((row) =>
+ rawCandidateRowSchema.parse(row),
+ );
this.logger?.info(
{
durationMs: Math.round((performance.now() - startedAt) * 100) / 100,
diff --git a/apps/server/src/infrastructure/ai/remote-crop-request.ts b/apps/server/src/infrastructure/ai/remote-crop-request.ts
new file mode 100644
index 000000000..29725743a
--- /dev/null
+++ b/apps/server/src/infrastructure/ai/remote-crop-request.ts
@@ -0,0 +1,10 @@
+export function createRemoteCropRequest(
+ fileBuffer: Uint8Array,
+ fileName: string,
+ transparent: boolean,
+): { file: File; transparent: boolean } {
+ return {
+ file: new File([Uint8Array.from(fileBuffer).buffer], fileName),
+ transparent,
+ };
+}
diff --git a/apps/server/src/infrastructure/ai/rust-ai-client.ts b/apps/server/src/infrastructure/ai/rust-ai-client.ts
index ef69c2d76..386f065c6 100644
--- a/apps/server/src/infrastructure/ai/rust-ai-client.ts
+++ b/apps/server/src/infrastructure/ai/rust-ai-client.ts
@@ -167,33 +167,47 @@ export class RustAiClient implements IAiClient {
}
}
- async tagImage(imageBuffer: ArrayBuffer): Promise {
+ async tagImage(
+ imageBuffer: ArrayBuffer,
+ signal?: AbortSignal,
+ ): Promise {
+ signal?.throwIfAborted();
if (this.baseUrl) {
- const result = await this.callRemoteOrpcWithFile(
- (c, f) => c.ai.tag({ file: f }),
- imageBuffer,
+ const result = await abortable(
+ this.callRemoteOrpcWithFile(
+ (c, f) => c.ai.tag({ file: f }),
+ imageBuffer,
+ ),
+ signal,
);
return taggingResponseSchema.parse(result);
}
return this.withTempFile(imageBuffer, "rust-tag", (filePath) =>
- this.tagImageByPath(filePath),
+ this.tagImageByPath(filePath, signal),
);
}
- async tagImageByPath(filePath: string): Promise {
+ async tagImageByPath(
+ filePath: string,
+ signal?: AbortSignal,
+ ): Promise {
+ signal?.throwIfAborted();
if (this.baseUrl) {
const buffer = await Bun.file(filePath).bytes();
- const result = await this.callRemoteOrpcWithFile(
- (c, f) => c.ai.tag({ file: f }),
- buffer,
- path.basename(filePath),
+ const result = await abortable(
+ this.callRemoteOrpcWithFile(
+ (c, f) => c.ai.tag({ file: f }),
+ buffer,
+ path.basename(filePath),
+ ),
+ signal,
);
return taggingResponseSchema.parse(result);
}
const { getPixaiTags } = await import("dghs-imgutils-rs");
- const result = await getPixaiTags(filePath);
+ const result = await abortable(getPixaiTags(filePath), signal);
return taggingResponseSchema.parse({
general: result.general,
character: result.character,
@@ -226,35 +240,45 @@ export class RustAiClient implements IAiClient {
async extractCcipFeature(
imageBuffer: ArrayBuffer,
+ signal?: AbortSignal,
): Promise {
+ signal?.throwIfAborted();
if (this.baseUrl) {
- const result = await this.callRemoteOrpcWithFile(
- (c, f) => c.ai.ccipFeature({ file: f }),
- imageBuffer,
+ const result = await abortable(
+ this.callRemoteOrpcWithFile(
+ (c, f) => c.ai.ccipFeature({ file: f }),
+ imageBuffer,
+ ),
+ signal,
);
return ccipFeatureResponseSchema.parse(result);
}
return this.withTempFile(imageBuffer, "rust-ccip", (filePath) =>
- this.extractCcipFeatureByPath(filePath),
+ this.extractCcipFeatureByPath(filePath, signal),
);
}
async extractCcipFeatureByPath(
filePath: string,
+ signal?: AbortSignal,
): Promise {
+ signal?.throwIfAborted();
if (this.baseUrl) {
const buffer = await Bun.file(filePath).bytes();
- const result = await this.callRemoteOrpcWithFile(
- (c, f) => c.ai.ccipFeature({ file: f }),
- buffer,
- path.basename(filePath),
+ const result = await abortable(
+ this.callRemoteOrpcWithFile(
+ (c, f) => c.ai.ccipFeature({ file: f }),
+ buffer,
+ path.basename(filePath),
+ ),
+ signal,
);
return ccipFeatureResponseSchema.parse(result);
}
const { ccipGetEmbedding } = await import("dghs-imgutils-rs");
- const embedding = await ccipGetEmbedding(filePath);
+ const embedding = await abortable(ccipGetEmbedding(filePath), signal);
return ccipFeatureResponseSchema.parse({
feature: embedding,
});
@@ -317,3 +341,15 @@ export class RustAiClient implements IAiClient {
});
}
}
+
+async function abortable(promise: Promise, signal?: AbortSignal): Promise {
+ if (!signal) return await promise;
+ signal.throwIfAborted();
+ return await new Promise((resolve, reject) => {
+ const onAbort = () => reject(signal.reason);
+ signal.addEventListener("abort", onAbort, { once: true });
+ void promise.then(resolve, reject).finally(() => {
+ signal.removeEventListener("abort", onAbort);
+ });
+ });
+}
diff --git a/apps/server/src/infrastructure/api-clients/ai-api.ts b/apps/server/src/infrastructure/api-clients/ai-api.ts
index 402b2913b..aba995175 100644
--- a/apps/server/src/infrastructure/api-clients/ai-api.ts
+++ b/apps/server/src/infrastructure/api-clients/ai-api.ts
@@ -4,6 +4,11 @@
* NOTE: Migrated to use oRPC ✅
*/
+import type {
+ CreateManualMediaRegion,
+ SafeMediaRegion,
+ UpdateMediaRegion,
+} from "@solid-imager/core/domain/media-regions/schemas";
import type { tagImageRequestSchema } from "@solid-imager/core/domain/tagging/schemas";
import type { z } from "zod";
import { orpc } from "~/infrastructure/api-clients/orpc-client";
@@ -35,6 +40,48 @@ export function fetchCharacterCrops(mediaId: string, transparent: boolean) {
return orpc.ai.detectAndCropCharacters({ mediaId, transparent });
}
+export function fetchMediaRegions(mediaId: string) {
+ return orpc.mediaRegions.list({ mediaId });
+}
+
+export function createManualMediaRegion(input: CreateManualMediaRegion) {
+ return orpc.mediaRegions.createManual(input);
+}
+
+export function updateMediaRegion(input: UpdateMediaRegion) {
+ return orpc.mediaRegions.update(input);
+}
+
+export async function deleteMediaRegion(
+ regionId: string,
+ expectedRevision: string,
+): Promise {
+ await orpc.mediaRegions.delete({ regionId, expectedRevision });
+}
+
+export function materializeMediaRegion(
+ regionId: string,
+ expectedRevision: string,
+ transparent: boolean,
+) {
+ return orpc.mediaRegions.materialize({
+ regionId,
+ expectedRevision,
+ profile: { transparent },
+ });
+}
+
+export function getMediaRegionRenderUrl(
+ region: SafeMediaRegion,
+ transparent: boolean,
+): string {
+ const query = new URLSearchParams({
+ revision: region.regionRevision,
+ transparent: String(transparent),
+ });
+ return `/api/media-regions/${encodeURIComponent(region.id)}/render?${query}`;
+}
+
export function scanBatchTaggingTargets(params: {
force?: boolean;
mediaSourceId?: string;
diff --git a/apps/server/src/infrastructure/api/media-region-api-errors.ts b/apps/server/src/infrastructure/api/media-region-api-errors.ts
new file mode 100644
index 000000000..9d040eb7d
--- /dev/null
+++ b/apps/server/src/infrastructure/api/media-region-api-errors.ts
@@ -0,0 +1,22 @@
+import { ORPCError } from "@orpc/server";
+import {
+ ResourceConflictError,
+ ResourceNotFoundError,
+ ValidationError,
+} from "@solid-imager/core/domain/errors";
+import { z } from "zod";
+
+export function toMediaRegionOrpcError(error: unknown): never {
+ if (error instanceof ResourceNotFoundError) {
+ throw new ORPCError("NOT_FOUND", { message: error.message });
+ }
+ if (error instanceof ResourceConflictError) {
+ throw new ORPCError("CONFLICT", { message: error.message });
+ }
+ if (error instanceof ValidationError || error instanceof z.ZodError) {
+ throw new ORPCError("BAD_REQUEST", {
+ message: error instanceof Error ? error.message : "Invalid media region",
+ });
+ }
+ throw error;
+}
diff --git a/apps/server/src/infrastructure/api/media-region-render-handler.ts b/apps/server/src/infrastructure/api/media-region-render-handler.ts
new file mode 100644
index 000000000..454b8a0cb
--- /dev/null
+++ b/apps/server/src/infrastructure/api/media-region-render-handler.ts
@@ -0,0 +1,63 @@
+import type { MediaRegionService } from "@solid-imager/application/services/media-region-service";
+import {
+ ResourceConflictError,
+ ResourceNotFoundError,
+ ValidationError,
+} from "@solid-imager/core/domain/errors";
+import { mediaRevisionSchema } from "@solid-imager/core/domain/media-regions/schemas";
+
+export type MediaRegionRenderService = Pick<
+ MediaRegionService,
+ "getRenderIdentity" | "render"
+>;
+
+export async function handleMediaRegionRenderRequest(
+ request: Request,
+ regionId: string,
+ service: MediaRegionRenderService,
+): Promise {
+ const search = new URL(request.url).searchParams;
+ const parsedRevision = mediaRevisionSchema.safeParse(search.get("revision"));
+ if (!parsedRevision.success) {
+ return new Response("Invalid or missing region revision", { status: 400 });
+ }
+ const expectedRevision = parsedRevision.data;
+ const transparent = search.get("transparent") === "true";
+ try {
+ const { etag } = await service.getRenderIdentity(
+ regionId,
+ expectedRevision,
+ { transparent },
+ );
+ if (request.headers.get("If-None-Match") === etag) {
+ return new Response(null, {
+ status: 304,
+ headers: {
+ "Cache-Control": "private, no-cache",
+ ETag: etag,
+ },
+ });
+ }
+ const rendered = await service.render(regionId, expectedRevision, {
+ transparent,
+ });
+ return new Response(Uint8Array.from(rendered.bytes).buffer, {
+ headers: {
+ "Cache-Control": "private, no-cache",
+ "Content-Type": `image/${rendered.format}`,
+ ETag: etag,
+ },
+ });
+ } catch (error) {
+ if (error instanceof ResourceNotFoundError) {
+ return new Response(error.message, { status: 404 });
+ }
+ if (error instanceof ResourceConflictError) {
+ return new Response(error.message, { status: 409 });
+ }
+ if (error instanceof ValidationError) {
+ return new Response(error.message, { status: 400 });
+ }
+ throw error;
+ }
+}
diff --git a/apps/server/src/infrastructure/api/routers/ai-router.ts b/apps/server/src/infrastructure/api/routers/ai-router.ts
index e16c817c3..569337928 100644
--- a/apps/server/src/infrastructure/api/routers/ai-router.ts
+++ b/apps/server/src/infrastructure/api/routers/ai-router.ts
@@ -6,7 +6,8 @@ import {
CCIP_MODEL,
} from "@solid-imager/application/services/ccip-vector-service";
import { createClient } from "@solid-imager/client";
-
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
+import { getSafeJobErrorMessage } from "@solid-imager/core/domain/jobs/schemas";
import {
batchCcipExtractionRequestSchema,
batchTaggingRequestSchema,
@@ -31,6 +32,7 @@ import { services } from "~/application/registry";
import { ccipVectorService } from "~/application/services/ccip-vector-service";
import { taggingService } from "~/application/services/tagging-service";
import type { appRouter } from "~/domain/shared/api-contract";
+import { createRemoteCropRequest } from "~/infrastructure/ai/remote-crop-request";
import { db } from "~/infrastructure/db";
import {
jobs,
@@ -40,6 +42,7 @@ import {
medias,
mediaTags,
} from "~/infrastructure/db/schema";
+import { ccipJobTargetsMedia } from "~/infrastructure/jobs/ccip-job-query";
import { logger } from "~/infrastructure/logger";
function isRemoteServerLocal(url: string): boolean {
@@ -102,10 +105,12 @@ async function callRemoteCrop(
fileBuffer: Buffer,
fileName: string,
timeoutMs: number,
+ transparent: boolean,
): Promise {
- const file = new File([new Uint8Array(fileBuffer)], fileName);
const remoteOrpc = createRemoteOprcClient(remoteUrl, timeoutMs);
- return remoteOrpc.ai.detectAndCropCharacters({ file });
+ return remoteOrpc.ai.detectAndCropCharacters(
+ createRemoteCropRequest(fileBuffer, fileName, transparent),
+ );
}
async function cropDetection(
@@ -455,29 +460,29 @@ export const aiRouter = {
const { mediaSourceId, force, batchSize } = input;
const jobRepo = services.getJobRepository();
- const parentJob = await jobRepo.create({
- type: "bulk_tagging_parent",
- status: "in_progress",
- mediaSourceId,
- payload: {
- total: 0,
- processed: 0,
- failed: 0,
+ const parentJob = await jobRepo.createParentWithDispatch(
+ {
+ type: "bulk_tagging_parent",
+ status: "in_progress",
mediaSourceId,
- force,
+ payload: {
+ total: 0,
+ processed: 0,
+ failed: 0,
+ mediaSourceId,
+ force,
+ },
},
- });
-
- await jobRepo.create({
- type: "bulk_tagging_dispatch",
- mediaSourceId,
- parentId: parentJob.id,
- payload: {
+ {
+ type: "bulk_tagging_dispatch",
mediaSourceId,
- force,
- ...(batchSize !== undefined ? { batchSize } : {}),
+ payload: {
+ mediaSourceId,
+ force,
+ ...(batchSize !== undefined ? { batchSize } : {}),
+ },
},
- });
+ );
logger.info(
{
@@ -510,7 +515,7 @@ export const aiRouter = {
where: and(
eq(jobs.type, "extract_ccip_vector"),
eq(jobs.mediaSourceId, input.mediaSourceId),
- sql`${jobs.payload}->>'mediaId' = ${input.mediaId}`,
+ ccipJobTargetsMedia(input.mediaId),
),
orderBy: desc(jobs.createdAt),
});
@@ -527,7 +532,9 @@ export const aiRouter = {
return {
status: "failed" as const,
jobId: latestJob.id,
- error: latestJob.error ?? "CCIP vector extraction failed",
+ error:
+ getSafeJobErrorMessage(latestJob.errorCode) ??
+ "CCIP vector extraction failed",
};
}
return status;
@@ -551,9 +558,37 @@ export const aiRouter = {
.input(ccipExtractionRequestSchema)
.output(startCcipExtractionResponseSchema)
.handler(async ({ input }) => {
+ const [media] = await db
+ .select({
+ id: medias.id,
+ mediaSourceId: medias.mediaSourceId,
+ modifiedAt: medias.modifiedAt,
+ fileSize: medias.fileSize,
+ width: medias.width,
+ height: medias.height,
+ })
+ .from(medias)
+ .where(
+ and(
+ eq(medias.id, input.mediaId),
+ eq(medias.mediaSourceId, input.mediaSourceId),
+ ),
+ )
+ .limit(1);
+ if (!media) throw new Error("Media not found");
+ const inputRevision = await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
const job = await services.getJobRepository().create({
type: "extract_ccip_vector",
mediaSourceId: input.mediaSourceId,
+ targetId: input.mediaId,
+ inputRevision,
payload: { mediaId: input.mediaId, force: input.force },
});
logger.info(
@@ -617,27 +652,28 @@ export const aiRouter = {
.handler(async ({ input }) => {
const { mediaSourceId, force } = input;
const jobRepo = services.getJobRepository();
- const parent = await jobRepo.create({
- type: "batch_ccip_parent",
- status: "in_progress",
- mediaSourceId,
- payload: {
- total: 0,
- processed: 0,
- failed: 0,
+ const parent = await jobRepo.createParentWithDispatch(
+ {
+ type: "batch_ccip_parent",
+ status: "in_progress",
mediaSourceId,
- force,
+ payload: {
+ total: 0,
+ processed: 0,
+ failed: 0,
+ mediaSourceId,
+ force,
+ },
},
- });
- await jobRepo.create({
- type: "batch_ccip_dispatch",
- mediaSourceId,
- parentId: parent.id,
- payload: {
+ {
+ type: "batch_ccip_dispatch",
mediaSourceId,
- force,
+ payload: {
+ mediaSourceId,
+ force,
+ },
},
- });
+ );
logger.info(
{
jobId: parent.id,
@@ -666,6 +702,7 @@ export const aiRouter = {
}),
]),
)
+ .output(detectAndCropResponseSchema)
.handler(async ({ input }) => {
const startedAt = Date.now();
const logContext =
@@ -710,7 +747,10 @@ export const aiRouter = {
},
"Character detection and cropping completed",
);
- return { detections: resultDetections };
+ return {
+ mode: "file-preview" as const,
+ detections: resultDetections,
+ };
} finally {
await Bun.file(tmpPath)
.delete()
@@ -754,8 +794,24 @@ export const aiRouter = {
fileBuffer,
path.basename(fullPath),
config.ai.timeoutMs,
+ transparent,
),
);
+ if (result.mode !== "file-preview") {
+ throw new Error(
+ "Remote crop service returned an invalid response mode",
+ );
+ }
+ const regions = await services
+ .getMediaRegionService()
+ .persistDetections({
+ mediaId,
+ detections: result.detections.map((detection) => ({
+ bbox: detection.bbox,
+ label: detection.label,
+ score: detection.score,
+ })),
+ });
logger.info(
{
...logContext,
@@ -765,28 +821,26 @@ export const aiRouter = {
},
"Character detection and cropping completed",
);
- return result;
+ return { mode: "media-backed" as const, regions };
}
const { detectPerson } = await import("dghs-imgutils-rs");
const detections = await detectPerson(fullPath);
- const resultDetections = await Promise.all(
- detections.map(async (det, idx) =>
- cropDetection(fullPath, det, idx, transparent),
- ),
- );
+ const regions = await services
+ .getMediaRegionService()
+ .persistDetections({ mediaId, detections });
logger.info(
{
...logContext,
execution: "local",
- detectionCount: resultDetections.length,
+ detectionCount: regions.length,
durationMs: Date.now() - startedAt,
},
"Character detection and cropping completed",
);
- return { detections: resultDetections };
+ return { mode: "media-backed" as const, regions };
} catch (error) {
logger.error(
{
diff --git a/apps/server/src/infrastructure/api/routers/imports-router.ts b/apps/server/src/infrastructure/api/routers/imports-router.ts
index fae276591..9d275c282 100644
--- a/apps/server/src/infrastructure/api/routers/imports-router.ts
+++ b/apps/server/src/infrastructure/api/routers/imports-router.ts
@@ -1,4 +1,5 @@
import { eventIterator, os } from "@orpc/server";
+import { prepareJob } from "@solid-imager/core/domain/jobs/registry";
import { downloadItemSchema } from "@solid-imager/core/domain/media/schemas";
import {
type ImportEvent,
@@ -108,7 +109,7 @@ export const bulkAddHandler = async ({
const ChunkSize = 100;
for (let i = 0; i < jobValues.length; i += ChunkSize) {
const chunk = jobValues.slice(i, i + ChunkSize);
- await db.insert(jobs).values(chunk);
+ await db.insert(jobs).values(chunk.map(prepareJob)).onConflictDoNothing();
}
addedCount = importItems.length;
diff --git a/apps/server/src/infrastructure/api/routers/jobs-router.ts b/apps/server/src/infrastructure/api/routers/jobs-router.ts
index 7451f399e..f952502fd 100644
--- a/apps/server/src/infrastructure/api/routers/jobs-router.ts
+++ b/apps/server/src/infrastructure/api/routers/jobs-router.ts
@@ -1,11 +1,25 @@
import { eventIterator, os } from "@orpc/server";
+import {
+ getSafeJobErrorMessage,
+ type SafeJob,
+} from "@solid-imager/core/domain/jobs/schemas";
+import type { Job } from "@solid-imager/core/domain/repositories/job-repository";
+import { batchParentPayloadSchema } from "@solid-imager/core/domain/tagging/schemas";
import {
type JobEvent,
jobEventSchema,
} from "@solid-imager/core/domain/sources/events";
+import { z } from "zod";
+import { services } from "~/application/registry";
import { RealtimeEventBus } from "~/infrastructure/events/realtime-event-bus";
export const jobsRouter = {
+ get: os
+ .input(z.object({ id: z.string().uuid() }))
+ .handler(async ({ input }): Promise => {
+ const job = await services.getJobRepository().findById(input.id);
+ return job ? toSafeJob(job) : null;
+ }),
events: os.output(eventIterator(jobEventSchema)).handler(async function* ({
signal,
}) {
@@ -52,3 +66,32 @@ export const jobsRouter = {
}
}),
};
+
+export function toSafeJob(job: Job): SafeJob {
+ const progress = batchParentPayloadSchema.safeParse(job.payload);
+ return {
+ id: job.id,
+ type: job.type,
+ status: job.status,
+ queueName: job.queueName,
+ targetId: job.targetId,
+ inputRevision: job.inputRevision,
+ attemptCount: job.attemptCount,
+ maxAttempts: job.maxAttempts,
+ errorCode: job.errorCode,
+ errorMessage: getSafeJobErrorMessage(job.errorCode),
+ progress:
+ job.type === "bulk_tagging_parent" || job.type === "batch_ccip_parent"
+ ? progress.success
+ ? {
+ processed: progress.data.processed,
+ failed: progress.data.failed,
+ total: progress.data.total,
+ }
+ : { processed: 0, failed: 0, total: 0 }
+ : null,
+ createdAt: job.createdAt,
+ updatedAt: job.updatedAt,
+ parentId: job.parentId,
+ };
+}
diff --git a/apps/server/src/infrastructure/api/routers/media-regions-router.ts b/apps/server/src/infrastructure/api/routers/media-regions-router.ts
new file mode 100644
index 000000000..b716fbdd9
--- /dev/null
+++ b/apps/server/src/infrastructure/api/routers/media-regions-router.ts
@@ -0,0 +1,70 @@
+import { os } from "@orpc/server";
+import {
+ createManualMediaRegionSchema,
+ deleteMediaRegionSchema,
+ materializedMediaRegionSchema,
+ materializeMediaRegionSchema,
+ safeMediaRegionSchema,
+ updateMediaRegionSchema,
+} from "@solid-imager/core/domain/media-regions/schemas";
+import { z } from "zod";
+import { services } from "~/application/registry";
+import { toMediaRegionOrpcError } from "~/infrastructure/api/media-region-api-errors";
+
+export const mediaRegionsRouter = {
+ list: os
+ .input(z.object({ mediaId: z.string().uuid() }))
+ .output(z.array(safeMediaRegionSchema))
+ .handler(async ({ input }) => {
+ try {
+ return await services.getMediaRegionService().list(input.mediaId);
+ } catch (error) {
+ return toMediaRegionOrpcError(error);
+ }
+ }),
+ createManual: os
+ .input(createManualMediaRegionSchema)
+ .output(safeMediaRegionSchema)
+ .handler(async ({ input }) => {
+ try {
+ return await services.getMediaRegionService().createManual(input);
+ } catch (error) {
+ return toMediaRegionOrpcError(error);
+ }
+ }),
+ update: os
+ .input(updateMediaRegionSchema)
+ .output(safeMediaRegionSchema)
+ .handler(async ({ input }) => {
+ try {
+ return await services.getMediaRegionService().update(input);
+ } catch (error) {
+ return toMediaRegionOrpcError(error);
+ }
+ }),
+ delete: os
+ .input(deleteMediaRegionSchema)
+ .output(z.object({ success: z.literal(true) }))
+ .handler(async ({ input }) => {
+ try {
+ await services
+ .getMediaRegionService()
+ .delete(input.regionId, input.expectedRevision);
+ return { success: true as const };
+ } catch (error) {
+ return toMediaRegionOrpcError(error);
+ }
+ }),
+ materialize: os
+ .input(materializeMediaRegionSchema)
+ .output(materializedMediaRegionSchema)
+ .handler(async ({ input }) => {
+ try {
+ return await services
+ .getMediaRegionService()
+ .materialize(input.regionId, input.expectedRevision, input.profile);
+ } catch (error) {
+ return toMediaRegionOrpcError(error);
+ }
+ }),
+};
diff --git a/apps/server/src/infrastructure/bootstrap.ts b/apps/server/src/infrastructure/bootstrap.ts
index f0c2a7591..1e631f249 100644
--- a/apps/server/src/infrastructure/bootstrap.ts
+++ b/apps/server/src/infrastructure/bootstrap.ts
@@ -1,3 +1,4 @@
+import { MediaRegionService } from "@solid-imager/application/services/media-region-service";
import { services } from "~/application/registry";
import { configureCcipVectorService } from "~/application/services/ccip-vector-service";
import { CharacterServiceImpl } from "~/application/services/character-service";
@@ -14,10 +15,12 @@ import { JobWorker } from "~/infrastructure/jobs/job-worker";
import { generateThumbnail } from "~/infrastructure/jobs/thumbnails";
import { logger, updateLogLevel } from "~/infrastructure/logger";
import { ImageProcessor } from "~/infrastructure/processing/image-processor";
+import { SharpMediaRegionRenderer } from "~/infrastructure/processing/media-region-renderer";
import { AuthorRepository } from "~/infrastructure/repositories/author-repository";
import { DrizzleCharacterRepository } from "~/infrastructure/repositories/character-repository";
import { IpRepository } from "~/infrastructure/repositories/ip-repository";
import { JobRepository } from "~/infrastructure/repositories/job-repository";
+import { DrizzleMediaRegionRepository } from "~/infrastructure/repositories/media-region-repository";
import { MediaRepository } from "~/infrastructure/repositories/media-repository";
import { ProjectRepository } from "~/infrastructure/repositories/project-repository";
import { DrizzleSourceRepository as ActualSourceRepo } from "~/infrastructure/repositories/source-repository";
@@ -56,6 +59,7 @@ export function initServices() {
// Register Repositories
services.registerMediaRepository(MediaRepository);
+ services.registerMediaRegionRepository(DrizzleMediaRegionRepository);
services.registerSourceRepository(ActualSourceRepo);
services.registerTagRepository(TagRepository);
services.registerAuthorRepository(AuthorRepository);
@@ -70,6 +74,16 @@ export function initServices() {
services.registerMediaStorage(ServerMediaStorage);
services.registerFileSystem(new NodeFileSystem());
services.registerImageProcessor(ImageProcessor);
+ services.registerMediaRegionService(
+ new MediaRegionService({
+ regionRepository: DrizzleMediaRegionRepository,
+ mediaRepository: MediaRepository,
+ sourceRepository: ActualSourceRepo,
+ transactionManager: DrizzleTransactionManager,
+ mediaStorage: ServerMediaStorage,
+ renderer: new SharpMediaRegionRenderer(ActualSourceRepo),
+ }),
+ );
// Initialize RustAiClient with config values
const rustAiClient = new RustAiClient(config.ai.baseUrl, config.ai.timeoutMs);
diff --git a/apps/server/src/infrastructure/jobs/ccip-job-query.ts b/apps/server/src/infrastructure/jobs/ccip-job-query.ts
new file mode 100644
index 000000000..fa193ca89
--- /dev/null
+++ b/apps/server/src/infrastructure/jobs/ccip-job-query.ts
@@ -0,0 +1,14 @@
+import { or, sql, type SQL } from "drizzle-orm";
+import { jobs } from "~/infrastructure/db/schema";
+
+/** Matches both current one-media child jobs and legacy mediaIds batch jobs. */
+export function ccipJobTargetsMedia(mediaId: string): SQL {
+ const condition = or(
+ sql`${jobs.payload}->>'mediaId' = ${mediaId}`,
+ sql`(${jobs.payload}->'mediaIds') ? ${mediaId}`,
+ );
+ if (!condition) {
+ throw new Error("CCIP job target condition could not be constructed");
+ }
+ return condition;
+}
diff --git a/apps/server/src/infrastructure/jobs/ccip-jobs.ts b/apps/server/src/infrastructure/jobs/ccip-jobs.ts
index fb86bce74..60b51e513 100644
--- a/apps/server/src/infrastructure/jobs/ccip-jobs.ts
+++ b/apps/server/src/infrastructure/jobs/ccip-jobs.ts
@@ -3,7 +3,12 @@ import {
CCIP_EMBEDDING_VERSION,
CCIP_MODEL,
} from "@solid-imager/application/services/ccip-vector-service";
-import { batchParentPayloadSchema } from "@solid-imager/core/domain/tagging/schemas";
+import { prepareJob } from "@solid-imager/core/domain/jobs/registry";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
+import type {
+ Job,
+ NewJob,
+} from "@solid-imager/core/domain/repositories/job-repository";
import { getErrorMessage } from "@solid-imager/core/utils";
import { and, asc, eq, gt, notExists, or, sql } from "drizzle-orm";
import { z } from "zod";
@@ -11,15 +16,27 @@ import { services } from "~/application/registry";
import { ccipVectorService } from "~/application/services/ccip-vector-service";
import { db } from "~/infrastructure/db";
import {
- type Job,
jobs,
mediaSources,
medias,
- type NewJob,
} from "~/infrastructure/db/schema";
import { RealtimeEventBus } from "~/infrastructure/events/realtime-event-bus";
import { logger } from "~/infrastructure/logger";
+type ExecutableJob = Pick<
+ Job,
+ | "id"
+ | "type"
+ | "mediaSourceId"
+ | "status"
+ | "payload"
+ | "result"
+ | "error"
+ | "createdAt"
+ | "updatedAt"
+ | "parentId"
+>;
+
const singleExtractionPayloadSchema = z.object({
mediaId: z.string().uuid(),
force: z.boolean().default(false),
@@ -41,30 +58,11 @@ const batchCcipDispatchPayloadSchema = z.object({
mediaSourceId: z.string().uuid().optional(),
});
-const EXTRACTION_JOB_BATCH_SIZE = 25;
const CHILD_INSERT_CHUNK = 500;
-async function finalizeBatchParent(
- parentId: string,
- progress: { processed: number; failed: number; total: number },
+export async function processBatchCcipDispatchJob(
+ job: ExecutableJob,
): Promise {
- const jobRepo = services.getJobRepository();
- if (progress.failed > 0) {
- await jobRepo.update(parentId, { status: "failed" });
- RealtimeEventBus.publishJob("job-failed", {
- jobId: parentId,
- error: `${progress.failed} item(s) failed`,
- });
- return;
- }
- await jobRepo.update(parentId, { status: "completed" });
- RealtimeEventBus.publishJob("job-completed", {
- jobId: parentId,
- message: "CCIP vector extraction completed",
- });
-}
-
-export async function processBatchCcipDispatchJob(job: Job): Promise {
const payload = batchCcipDispatchPayloadSchema.parse(job.payload);
const force = payload.force ?? false;
const batchSize = payload.batchSize ?? 1000;
@@ -108,6 +106,9 @@ export async function processBatchCcipDispatchJob(job: Job): Promise {
id: medias.id,
mediaSourceId: medias.mediaSourceId,
modifiedAt: medias.modifiedAt,
+ fileSize: medias.fileSize,
+ width: medias.width,
+ height: medias.height,
})
.from(medias)
.innerJoin(mediaSources, eq(mediaSources.id, medias.mediaSourceId))
@@ -154,19 +155,23 @@ export async function processBatchCcipDispatchJob(job: Job): Promise {
}
const jobRows: NewJob[] = [];
for (const [sourceId, sourceRows] of rowsBySource) {
- for (
- let index = 0;
- index < sourceRows.length;
- index += EXTRACTION_JOB_BATCH_SIZE
- ) {
+ for (const row of sourceRows) {
+ const inputRevision = await createMediaSourceRevision({
+ mediaId: row.id,
+ mediaSourceId: row.mediaSourceId,
+ modifiedAt: row.modifiedAt,
+ fileSize: row.fileSize,
+ width: row.width,
+ height: row.height,
+ });
jobRows.push({
type: "extract_ccip_vector",
mediaSourceId: sourceId,
parentId,
+ targetId: row.id,
+ inputRevision,
payload: {
- mediaIds: sourceRows
- .slice(index, index + EXTRACTION_JOB_BATCH_SIZE)
- .map((row) => row.id),
+ mediaId: row.id,
force,
},
});
@@ -174,7 +179,7 @@ export async function processBatchCcipDispatchJob(job: Job): Promise {
}
for (let i = 0; i < jobRows.length; i += CHILD_INSERT_CHUNK) {
const chunk = jobRows.slice(i, i + CHILD_INSERT_CHUNK);
- await db.insert(jobs).values(chunk);
+ await db.insert(jobs).values(chunk.map(prepareJob)).onConflictDoNothing();
}
dispatchedCount += targetRows.length;
@@ -191,36 +196,8 @@ export async function processBatchCcipDispatchJob(job: Job): Promise {
}
const jobRepo = services.getJobRepository();
- const [childSummary] = await db
- .select({
- total: sql`COALESCE(SUM(
- CASE
- WHEN jsonb_typeof(${jobs.payload}->'mediaIds') = 'array'
- THEN jsonb_array_length(${jobs.payload}->'mediaIds')
- ELSE 1
- END
- ), 0)::int`,
- })
- .from(jobs)
- .where(
- and(eq(jobs.parentId, parentId), eq(jobs.type, "extract_ccip_vector")),
- );
- const total = Number(childSummary?.total ?? 0);
- const [updatedParent] = await db
- .update(jobs)
- .set({
- payload: sql`jsonb_set(
- COALESCE(${jobs.payload}, '{}'::jsonb),
- '{total}',
- to_jsonb(${total}::int)
- )`,
- updatedAt: new Date(),
- })
- .where(eq(jobs.id, parentId))
- .returning();
- const parentPayload = batchParentPayloadSchema.parse(
- updatedParent?.payload ?? {},
- );
+ const progress = await jobRepo.recomputeBatchProgress(parentId);
+ const total = progress?.total ?? 0;
if (total === 0) {
await jobRepo.update(parentId, { status: "completed" });
@@ -231,16 +208,9 @@ export async function processBatchCcipDispatchJob(job: Job): Promise {
} else {
RealtimeEventBus.publishJob("job-progress", {
jobId: parentId,
- processed: parentPayload.processed,
+ processed: progress?.processed ?? 0,
total,
});
- if (parentPayload.processed + parentPayload.failed >= total) {
- await finalizeBatchParent(parentId, {
- processed: parentPayload.processed,
- failed: parentPayload.failed,
- total,
- });
- }
}
logger.info(
@@ -249,14 +219,17 @@ export async function processBatchCcipDispatchJob(job: Job): Promise {
);
}
-export async function processCcipExtractionJob(job: Job): Promise {
+export async function processCcipExtractionJob(
+ job: ExecutableJob,
+ signal?: AbortSignal,
+): Promise {
const payload = extractionPayloadSchema.parse(job.payload);
if (!job.mediaSourceId) {
throw new Error("CCIP extraction job is missing mediaSourceId");
}
const mediaIds = "mediaIds" in payload ? payload.mediaIds : [payload.mediaId];
if (mediaIds.length > 1) {
- await processCcipExtractionBatch(job, mediaIds, payload.force);
+ await processCcipExtractionBatch(job, mediaIds, payload.force, signal);
return;
}
try {
@@ -265,6 +238,7 @@ export async function processCcipExtractionJob(job: Job): Promise {
job.mediaSourceId,
mediaId,
payload.force,
+ signal,
);
logger.info(
{
@@ -277,27 +251,18 @@ export async function processCcipExtractionJob(job: Job): Promise {
},
"CCIP vector extraction completed",
);
- RealtimeEventBus.publishJob("job-completed", {
- jobId: job.id,
- message: "CCIP vector extraction completed",
- });
-
- await updateParentProgress(job, 1, 0);
+ return result;
} catch (error) {
logger.error({ err: error, mediaIds }, "CCIP vector extraction failed");
- RealtimeEventBus.publishJob("job-failed", {
- jobId: job.id,
- error: getErrorMessage(error),
- });
- await updateParentProgress(job, 0, mediaIds.length);
throw error;
}
}
async function processCcipExtractionBatch(
- job: Job,
+ job: ExecutableJob,
mediaIds: string[],
force: boolean,
+ signal?: AbortSignal,
): Promise {
if (!job.mediaSourceId) {
throw new Error("CCIP extraction job is missing mediaSourceId");
@@ -309,27 +274,18 @@ async function processCcipExtractionBatch(
mediaIds,
force,
1,
+ signal,
);
} catch (error) {
logger.error(
{ err: error, mediaIds },
"CCIP vector extraction batch failed",
);
- RealtimeEventBus.publishJob("job-failed", {
- jobId: job.id,
- error: getErrorMessage(error),
- });
- await updateParentProgress(job, 0, mediaIds.length);
throw error;
}
- const processed = results.filter(
- (result) => result.status === "fulfilled",
- ).length;
const failures = results.filter(
(result): result is PromiseRejectedResult => result.status === "rejected",
);
- await updateParentProgress(job, processed, failures.length);
-
if (failures.length > 0) {
const error = new Error(
`${failures.length} of ${mediaIds.length} CCIP extraction(s) failed: ${getErrorMessage(failures[0].reason)}`,
@@ -338,10 +294,6 @@ async function processCcipExtractionBatch(
{ err: error, mediaIds },
"CCIP vector extraction batch failed",
);
- RealtimeEventBus.publishJob("job-failed", {
- jobId: job.id,
- error: error.message,
- });
throw error;
}
@@ -355,37 +307,4 @@ async function processCcipExtractionBatch(
},
"CCIP vector extraction batch completed",
);
- RealtimeEventBus.publishJob("job-completed", {
- jobId: job.id,
- message: `CCIP vector extraction completed (${mediaIds.length} items)`,
- });
-}
-
-async function updateParentProgress(
- job: Job,
- processed: number,
- failed: number,
-): Promise {
- if (!job.parentId) {
- return;
- }
- const jobRepo = services.getJobRepository();
- let progress = null;
- if (processed > 0) {
- progress = await jobRepo.incrementProgress(job.parentId, job.id, processed);
- }
- if (failed > 0) {
- progress = await jobRepo.incrementFailedCount(job.parentId, job.id, failed);
- }
- if (!progress || progress.total <= 0) {
- return;
- }
- RealtimeEventBus.publishJob("job-progress", {
- jobId: job.parentId,
- processed: progress.processed,
- total: progress.total,
- });
- if (progress.processed + progress.failed >= progress.total) {
- await finalizeBatchParent(job.parentId, progress);
- }
}
diff --git a/apps/server/src/infrastructure/jobs/download-jobs.ts b/apps/server/src/infrastructure/jobs/download-jobs.ts
index 54a8d5d98..243e6be50 100644
--- a/apps/server/src/infrastructure/jobs/download-jobs.ts
+++ b/apps/server/src/infrastructure/jobs/download-jobs.ts
@@ -5,6 +5,11 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
+import { prepareJob } from "@solid-imager/core/domain/jobs/registry";
+import type {
+ Job,
+ NewJob,
+} from "@solid-imager/core/domain/repositories/job-repository";
import type {
AddMediaRequest,
DownloadItem,
@@ -17,7 +22,7 @@ import { hasStderr, isRecord } from "@solid-imager/core/utils/type-guards";
import { create as createYtDlp, type Flags } from "youtube-dl-exec";
import { z } from "zod";
import { db } from "~/infrastructure/db";
-import { type Job, jobs, type NewJob } from "~/infrastructure/db/schema";
+import { jobs } from "~/infrastructure/db/schema";
import { RealtimeEventBus } from "~/infrastructure/events/realtime-event-bus";
import { waitForDownloadRateLimit } from "~/infrastructure/jobs/download-rate-limiter";
import { logger } from "~/infrastructure/logger";
@@ -886,7 +891,7 @@ export async function queueDownloadJobs(
const BATCH_SIZE = 500;
for (let i = 0; i < jobRows.length; i += BATCH_SIZE) {
const chunk = jobRows.slice(i, i + BATCH_SIZE);
- await db.insert(jobs).values(chunk);
+ await db.insert(jobs).values(chunk.map(prepareJob)).onConflictDoNothing();
}
// Jobs are picked up by the worker automatically.
diff --git a/apps/server/src/infrastructure/jobs/file-watcher-service.ts b/apps/server/src/infrastructure/jobs/file-watcher-service.ts
index 65e4a3127..c736d195f 100644
--- a/apps/server/src/infrastructure/jobs/file-watcher-service.ts
+++ b/apps/server/src/infrastructure/jobs/file-watcher-service.ts
@@ -5,6 +5,7 @@
*/
import path from "node:path";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
import { services } from "~/application/registry";
import { ccipVectorService } from "~/application/services/ccip-vector-service";
import { DirectorySyncService } from "~/application/services/directory-sync-service";
@@ -148,9 +149,19 @@ async function handleFileChanged(
// Queue processMedia job for thumbnail regeneration and metadata re-extraction
const jobRepo = services.getJobRepository();
+ const inputRevision = await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId,
+ modifiedAt: fileMetadata.modifiedAt,
+ fileSize: fileMetadata.size,
+ width: fileMetadata.width,
+ height: fileMetadata.height,
+ });
await jobRepo.create({
type: "processMedia",
mediaSourceId,
+ targetId: media.id,
+ inputRevision,
payload: {
mediaId: media.id,
sourcePath: basePath,
diff --git a/apps/server/src/infrastructure/jobs/job-errors.ts b/apps/server/src/infrastructure/jobs/job-errors.ts
new file mode 100644
index 000000000..896abb5bd
--- /dev/null
+++ b/apps/server/src/infrastructure/jobs/job-errors.ts
@@ -0,0 +1,9 @@
+export class NonRetryableJobError extends Error {
+ readonly code: string;
+
+ constructor(code: string, message: string) {
+ super(message);
+ this.name = "NonRetryableJobError";
+ this.code = code;
+ }
+}
diff --git a/apps/server/src/infrastructure/jobs/job-worker.ts b/apps/server/src/infrastructure/jobs/job-worker.ts
index 2f6879cfd..5de4354f3 100644
--- a/apps/server/src/infrastructure/jobs/job-worker.ts
+++ b/apps/server/src/infrastructure/jobs/job-worker.ts
@@ -1,7 +1,16 @@
-import type { AppConfig } from "@solid-imager/core/domain/config/config-schema";
-import type { IJobRepository } from "~/domain/repositories/job-repository";
-import type { Job } from "~/infrastructure/db/schema";
-import { logger } from "~/infrastructure/logger";
+import type { AppConfig } from '@solid-imager/core/domain/config/config-schema';
+import {
+ JOB_HEARTBEAT_MS,
+ retryDelayMs,
+} from '@solid-imager/core/domain/jobs/registry';
+import type {
+ ClaimFence,
+ IJobRepository,
+ Job,
+} from '@solid-imager/core/domain/repositories/job-repository';
+import { RealtimeEventBus } from '~/infrastructure/events/realtime-event-bus';
+import { NonRetryableJobError } from '~/infrastructure/jobs/job-errors';
+import { logger } from '~/infrastructure/logger';
type JsonSafeValue =
| string
@@ -15,9 +24,9 @@ function toJsonSafeValue(value: unknown): JsonSafeValue {
if (
value === null ||
value === undefined ||
- typeof value === "string" ||
- typeof value === "number" ||
- typeof value === "boolean"
+ typeof value === 'string' ||
+ typeof value === 'number' ||
+ typeof value === 'boolean'
) {
return value ?? null;
}
@@ -27,17 +36,17 @@ function toJsonSafeValue(value: unknown): JsonSafeValue {
if (Array.isArray(value)) {
return value.map(toJsonSafeValue);
}
- if (typeof value === "object") {
+ if (typeof value === 'object') {
const result: Record = {};
- for (const [key, val] of Object.entries(value)) {
- result[key] = toJsonSafeValue(val);
+ for (const [key, item] of Object.entries(value)) {
+ result[key] = toJsonSafeValue(item);
}
return result;
}
return null;
}
-const StaleInProgressJobMs = 60 * 60 * 1000;
+const LEASE_RECOVERY_INTERVAL_MS = 60 * 1000;
export class JobWorker {
private isRunning = false;
@@ -49,38 +58,37 @@ export class JobWorker {
private activeJobs = 0;
private activeAiJobs = 0;
private readonly activeLanceDbSyncKeys = new Set();
+ private readonly workerId = `worker-${globalThis.crypto.randomUUID()}`;
private readonly jobRepo: IJobRepository;
- private readonly processor: (job: Job) => Promise;
+ private readonly processor: (job: Job, signal?: AbortSignal) => Promise;
private readonly aiJobTypes = new Set([
- "auto_tagging",
- "extract_ccip_vector",
+ 'auto_tagging',
+ 'extract_ccip_vector',
]);
constructor(
jobRepo: IJobRepository,
- processor: (job: Job) => Promise,
+ processor: (job: Job, signal?: AbortSignal) => Promise,
) {
this.jobRepo = jobRepo;
this.processor = processor;
}
- start() {
- if (this.isRunning) {
- return;
- }
+ start(): void {
+ if (this.isRunning) return;
this.isRunning = true;
- logger.info("Job processing worker started");
- void this.recoverStaleJobs();
+ logger.info({ workerId: this.workerId }, 'Job processing worker started');
+ void this.recoverExpiredLeases();
this.recoverStaleJobsIntervalId = setInterval(
- () => void this.recoverStaleJobs(),
- 5 * 60 * 1000,
+ () => void this.recoverExpiredLeases(),
+ LEASE_RECOVERY_INTERVAL_MS,
);
- this.poll();
+ void this.poll();
}
- stop() {
+ stop(): void {
this.isRunning = false;
if (this.timeoutId) {
clearTimeout(this.timeoutId);
@@ -90,10 +98,10 @@ export class JobWorker {
clearInterval(this.recoverStaleJobsIntervalId);
this.recoverStaleJobsIntervalId = null;
}
- logger.info("Job processing worker stopped");
+ logger.info({ workerId: this.workerId }, 'Job processing worker stopped');
}
- updateConfig(config: AppConfig) {
+ updateConfig(config: AppConfig): void {
const oldConcurrency = this.concurrency;
const oldAiConcurrency = this.aiConcurrency;
const oldPollInterval = this.pollIntervalMs;
@@ -114,153 +122,242 @@ export class JobWorker {
aiConcurrency: this.aiConcurrency,
pollIntervalMs: this.pollIntervalMs,
},
- "JobWorker config updated",
+ 'JobWorker config updated',
);
}
}
- private async poll() {
- if (!this.isRunning) {
- return;
- }
+ private async poll(): Promise {
+ if (!this.isRunning) return;
try {
- // 1. Poll AI Jobs
if (this.activeAiJobs < this.aiConcurrency) {
const slots = this.aiConcurrency - this.activeAiJobs;
- if (slots > 0) {
- const jobs = await this.jobRepo.claimPending(slots, {
- includeTypes: Array.from(this.aiJobTypes),
- });
- for (const job of jobs) {
- void this.tryProcessJob(job);
- }
- }
+ const claimed = await this.jobRepo.claimPending(slots, {
+ includeTypes: [...this.aiJobTypes],
+ queueNames: ['ai'],
+ workerId: this.workerId,
+ });
+ for (const job of claimed) void this.tryProcessJob(job);
}
- // 2. Poll Other Jobs
- // "concurrency" is treated as the limit for NON-AI jobs in this independent pool model
const activeOtherJobs = this.activeJobs - this.activeAiJobs;
if (activeOtherJobs < this.concurrency) {
const slots = this.concurrency - activeOtherJobs;
- if (slots > 0) {
- const jobs = await this.jobRepo.claimPending(slots, {
- excludeTypes: Array.from(this.aiJobTypes),
- excludeLanceDbSourceIds: Array.from(this.activeLanceDbSyncKeys),
- });
- for (const job of jobs) {
- void this.tryProcessJob(job);
- }
- }
+ const claimed = await this.jobRepo.claimPending(slots, {
+ excludeTypes: [...this.aiJobTypes],
+ queueNames: ['default'],
+ excludeLanceDbSourceIds: [...this.activeLanceDbSyncKeys],
+ workerId: this.workerId,
+ });
+ for (const job of claimed) void this.tryProcessJob(job);
}
} catch (error) {
- logger.error({ err: error }, "Error polling for jobs");
+ logger.error({ err: error, workerId: this.workerId }, 'Error polling for jobs');
}
if (this.isRunning) {
- this.timeoutId = setTimeout(() => this.poll(), this.pollIntervalMs);
+ this.timeoutId = setTimeout(() => void this.poll(), this.pollIntervalMs);
}
}
- private async tryProcessJob(job: Job) {
+ private async tryProcessJob(job: Job): Promise {
+ const fence = getClaimFence(job);
+ if (!fence) {
+ logger.error(
+ { jobId: job.id, workerId: this.workerId },
+ 'Claimed job is missing a claim token',
+ );
+ return;
+ }
+
const lanceDbSyncKey = getLanceDbSyncKey(job);
- if (lanceDbSyncKey) {
- if (this.activeLanceDbSyncKeys.has(lanceDbSyncKey)) {
- logger.warn(
- { jobId: job.id, mediaSourceId: lanceDbSyncKey },
- "Requeueing overlapping claimed LanceDB sync job",
- );
- try {
- await this.jobRepo.update(job.id, { status: "pending" });
- } catch (error) {
- logger.error(
- { err: error, jobId: job.id },
- "Failed to requeue overlapping job",
- );
- }
- return;
- }
- this.activeLanceDbSyncKeys.add(lanceDbSyncKey);
+ if (lanceDbSyncKey && this.activeLanceDbSyncKeys.has(lanceDbSyncKey)) {
+ logger.warn(
+ { jobId: job.id, mediaSourceId: lanceDbSyncKey },
+ 'Releasing overlapping claimed LanceDB sync job',
+ );
+ await this.jobRepo.releaseClaim(job.id, fence);
+ return;
}
+ if (lanceDbSyncKey) this.activeLanceDbSyncKeys.add(lanceDbSyncKey);
- void this.processJob(job, lanceDbSyncKey);
+ await this.processJob(job, fence, lanceDbSyncKey);
}
- private async processJob(job: Job, lanceDbSyncKey?: string) {
+ private async processJob(
+ job: Job,
+ fence: ClaimFence,
+ lanceDbSyncKey?: string,
+ ): Promise {
this.activeJobs++;
const isAiJob = this.aiJobTypes.has(job.type);
+ if (isAiJob) this.activeAiJobs++;
const startedAt = Date.now();
- if (isAiJob) {
- this.activeAiJobs++;
- }
+ const abortController = new AbortController();
+ let heartbeatInFlight = false;
+ let leaseLost = false;
+ const heartbeatId = setInterval(() => {
+ if (heartbeatInFlight || leaseLost) return;
+ heartbeatInFlight = true;
+ void this.jobRepo
+ .heartbeatClaim(job.id, fence)
+ .then((accepted) => {
+ if (!accepted) {
+ leaseLost = true;
+ abortController.abort();
+ logger.warn(
+ { jobId: job.id, claimToken: fence.claimToken },
+ 'Job lease was lost; discarding worker output',
+ );
+ }
+ })
+ .catch((error) => {
+ logger.error({ err: error, jobId: job.id }, 'Job heartbeat failed');
+ })
+ .finally(() => {
+ heartbeatInFlight = false;
+ });
+ }, JOB_HEARTBEAT_MS);
logger.info(
{
jobId: job.id,
type: job.type,
+ attemptCount: job.attemptCount,
mediaSourceId: job.mediaSourceId,
parentId: job.parentId,
- isAiJob,
+ workerId: this.workerId,
},
- "Job started",
+ 'Job started',
);
+
try {
- const result = await this.processor(job);
+ const result = await this.processor(job, abortController.signal);
+ if (leaseLost || abortController.signal.aborted) return;
const safeResult =
result !== undefined ? toJsonSafeValue(result) : { success: true };
- await this.jobRepo.markAsCompleted(job.id, safeResult);
+ const accepted = await this.jobRepo.completeClaim(job.id, fence, safeResult);
+ if (!accepted) {
+ logger.warn(
+ { jobId: job.id, claimToken: fence.claimToken },
+ 'Completion was rejected by the job claim fence',
+ );
+ return;
+ }
+ RealtimeEventBus.publishJob('job-completed', {
+ jobId: job.id,
+ message: `${job.type} completed`,
+ });
+ await this.reconcileParent(job);
logger.info(
{
jobId: job.id,
type: job.type,
- mediaSourceId: job.mediaSourceId,
- parentId: job.parentId,
durationMs: Date.now() - startedAt,
},
- "Job completed",
+ 'Job completed',
);
} catch (error) {
- const errorMessage =
- error instanceof Error ? error.message : String(error);
+ if (leaseLost || abortController.signal.aborted) return;
+ const errorMessage = error instanceof Error ? error.message : String(error);
+ const nonRetryable = error instanceof NonRetryableJobError;
+ const failure = await this.jobRepo.failClaim(job.id, fence, {
+ error: errorMessage,
+ errorCode: nonRetryable ? error.code : 'JOB_EXECUTION_FAILED',
+ retryable: !nonRetryable,
+ retryAt: new Date(Date.now() + retryDelayMs(job.attemptCount)),
+ });
+ if (!failure) {
+ logger.warn(
+ { jobId: job.id, claimToken: fence.claimToken },
+ 'Failure was rejected by the job claim fence',
+ );
+ return;
+ }
logger.error(
{
err: error,
jobId: job.id,
type: job.type,
- mediaSourceId: job.mediaSourceId,
- parentId: job.parentId,
+ attemptCount: failure.attemptCount,
+ status: failure.status,
durationMs: Date.now() - startedAt,
},
- "Job failed",
+ failure.status === 'pending' ? 'Job scheduled for retry' : 'Job failed',
);
- await this.jobRepo.markAsFailed(job.id, errorMessage);
+ if (failure.status === 'failed') {
+ RealtimeEventBus.publishJob('job-failed', {
+ jobId: job.id,
+ error: errorMessage,
+ });
+ if (isDispatchJob(job) && job.parentId) {
+ await this.jobRepo.update(job.parentId, {
+ status: 'failed',
+ error: `Dispatch failed: ${errorMessage}`,
+ errorCode: 'DISPATCH_FAILED',
+ });
+ RealtimeEventBus.publishJob('job-failed', {
+ jobId: job.parentId,
+ error: `Dispatch failed: ${errorMessage}`,
+ });
+ } else {
+ await this.reconcileParent(job);
+ }
+ }
} finally {
+ clearInterval(heartbeatId);
this.activeJobs--;
- if (isAiJob) {
- this.activeAiJobs--;
- }
- if (lanceDbSyncKey) {
- this.activeLanceDbSyncKeys.delete(lanceDbSyncKey);
- }
+ if (isAiJob) this.activeAiJobs--;
+ if (lanceDbSyncKey) this.activeLanceDbSyncKeys.delete(lanceDbSyncKey);
+ }
+ }
+
+ private async reconcileParent(job: Job): Promise {
+ if (!job.parentId) return;
+ const reconciliation = await this.jobRepo.recomputeBatchProgress(job.parentId);
+ if (!reconciliation) return;
+ RealtimeEventBus.publishJob('job-progress', {
+ jobId: job.parentId,
+ processed: reconciliation.processed,
+ total: reconciliation.total,
+ });
+ if (!reconciliation.transitioned) return;
+ if (reconciliation.status === 'failed') {
+ RealtimeEventBus.publishJob('job-failed', {
+ jobId: job.parentId,
+ error: `${reconciliation.failed} child job(s) failed`,
+ });
+ } else if (reconciliation.status === 'completed') {
+ RealtimeEventBus.publishJob('job-completed', {
+ jobId: job.parentId,
+ message: 'Batch job completed',
+ });
}
}
- private async recoverStaleJobs() {
- const olderThan = new Date(Date.now() - StaleInProgressJobMs);
+
+ private async recoverExpiredLeases(): Promise {
try {
- const count = await this.jobRepo.requeueStaleInProgress(olderThan);
+ const count = await this.jobRepo.requeueExpiredLeases();
if (count > 0) {
- logger.warn({ count, olderThan }, "Requeued stale in-progress jobs");
+ logger.warn({ count }, 'Recovered expired job leases');
}
} catch (error) {
- logger.error({ err: error }, "Failed to requeue stale in-progress jobs");
+ logger.error({ err: error }, 'Failed to recover expired job leases');
}
}
}
+function getClaimFence(job: Job): ClaimFence | null {
+ return job.claimToken
+ ? { claimToken: job.claimToken, inputRevision: job.inputRevision }
+ : null;
+}
+
function getLanceDbSyncKey(job: Job): string | undefined {
if (
job.mediaSourceId &&
- ["sync_lancedb", "sync_lancedb_full", "sync_lancedb_delta"].includes(
+ ['sync_lancedb', 'sync_lancedb_full', 'sync_lancedb_delta'].includes(
job.type,
)
) {
@@ -268,3 +365,9 @@ function getLanceDbSyncKey(job: Job): string | undefined {
}
return undefined;
}
+
+function isDispatchJob(job: Job): boolean {
+ return (
+ job.type === 'bulk_tagging_dispatch' || job.type === 'batch_ccip_dispatch'
+ );
+}
diff --git a/apps/server/src/infrastructure/jobs/tagging-jobs.ts b/apps/server/src/infrastructure/jobs/tagging-jobs.ts
index f0f567811..9f847806a 100644
--- a/apps/server/src/infrastructure/jobs/tagging-jobs.ts
+++ b/apps/server/src/infrastructure/jobs/tagging-jobs.ts
@@ -1,21 +1,38 @@
-import { batchParentPayloadSchema } from "@solid-imager/core/domain/tagging/schemas";
+import { prepareJob } from "@solid-imager/core/domain/jobs/registry";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
+import type {
+ Job,
+ NewJob,
+} from "@solid-imager/core/domain/repositories/job-repository";
import { and, asc, eq, gt, notExists, sql } from "drizzle-orm";
import { z } from "zod";
import { services } from "~/application/registry";
import { taggingService } from "~/application/services/tagging-service";
import { db } from "~/infrastructure/db";
import {
- type Job,
jobs,
mediaCharacters,
mediaIps,
medias,
mediaTags,
- type NewJob,
} from "~/infrastructure/db/schema";
import { RealtimeEventBus } from "~/infrastructure/events/realtime-event-bus";
import { logger } from "~/infrastructure/logger";
+type ExecutableJob = Pick<
+ Job,
+ | "id"
+ | "type"
+ | "mediaSourceId"
+ | "status"
+ | "payload"
+ | "result"
+ | "error"
+ | "createdAt"
+ | "updatedAt"
+ | "parentId"
+>;
+
const autoTaggingPayloadSchema = z.object({
mediaId: z.string(),
force: z.boolean().optional(),
@@ -27,27 +44,10 @@ const bulkTaggingDispatchPayloadSchema = z.object({
mediaSourceId: z.string().optional(),
});
-async function finalizeBatchParent(
- parentId: string,
- progress: { processed: number; failed: number; total: number },
+export async function processAutoTaggingJob(
+ job: ExecutableJob,
+ signal?: AbortSignal,
): Promise {
- const jobRepo = services.getJobRepository();
- if (progress.failed > 0) {
- await jobRepo.update(parentId, { status: "failed" });
- RealtimeEventBus.publishJob("job-failed", {
- jobId: parentId,
- error: `${progress.failed} child job(s) failed`,
- });
- return;
- }
- await jobRepo.update(parentId, { status: "completed" });
- RealtimeEventBus.publishJob("job-completed", {
- jobId: parentId,
- message: "Batch tagging completed",
- });
-}
-
-export async function processAutoTaggingJob(job: Job): Promise {
const payload = autoTaggingPayloadSchema.parse(job.payload);
const { mediaId, force } = payload;
const { mediaSourceId, parentId } = job;
@@ -62,6 +62,7 @@ export async function processAutoTaggingJob(job: Job): Promise {
mediaId,
{
skipCache: force,
+ signal,
},
);
logger.info(
@@ -83,44 +84,15 @@ export async function processAutoTaggingJob(job: Job): Promise {
payload: { reason: "auto_tagging", mediaIds: [mediaId] },
});
- if (parentId) {
- const jobRepo = services.getJobRepository();
- const progress = await jobRepo.incrementProgress(parentId, job.id);
- if (!progress || progress.total === 0) {
- return;
- }
-
- RealtimeEventBus.publishJob("job-progress", {
- jobId: parentId,
- processed: progress.processed,
- total: progress.total,
- });
-
- if (progress.processed + progress.failed >= progress.total) {
- await finalizeBatchParent(parentId, progress);
- }
- }
} catch (error) {
logger.error({ err: error, mediaId }, "Auto tagging failed");
- if (parentId) {
- const jobRepo = services.getJobRepository();
- const progress = await jobRepo.incrementFailedCount(parentId, job.id);
- if (progress && progress.total > 0) {
- RealtimeEventBus.publishJob("job-progress", {
- jobId: parentId,
- processed: progress.processed,
- total: progress.total,
- });
- if (progress.processed + progress.failed >= progress.total) {
- await finalizeBatchParent(parentId, progress);
- }
- }
- }
throw error;
}
}
-export async function processBulkTaggingDispatchJob(job: Job): Promise {
+export async function processBulkTaggingDispatchJob(
+ job: ExecutableJob,
+): Promise {
const payload = bulkTaggingDispatchPayloadSchema.parse(job.payload);
const force = payload?.force ?? false;
const batchSize = payload?.batchSize ?? 1000;
@@ -198,6 +170,10 @@ export async function processBulkTaggingDispatchJob(job: Job): Promise {
.select({
id: medias.id,
mediaSourceId: medias.mediaSourceId,
+ modifiedAt: medias.modifiedAt,
+ fileSize: medias.fileSize,
+ width: medias.width,
+ height: medias.height,
})
.from(medias)
.where(
@@ -219,18 +195,29 @@ export async function processBulkTaggingDispatchJob(job: Job): Promise {
break;
}
- const jobRows: NewJob[] = results.map((row) => ({
- type: "auto_tagging",
- mediaSourceId: row.mediaSourceId,
- parentId,
- payload: {
- mediaId: row.id,
- force,
- },
- }));
+ const jobRows: NewJob[] = await Promise.all(
+ results.map(async (row) => ({
+ type: "auto_tagging",
+ mediaSourceId: row.mediaSourceId,
+ parentId,
+ targetId: row.id,
+ inputRevision: await createMediaSourceRevision({
+ mediaId: row.id,
+ mediaSourceId: row.mediaSourceId,
+ modifiedAt: row.modifiedAt,
+ fileSize: row.fileSize,
+ width: row.width,
+ height: row.height,
+ }),
+ payload: {
+ mediaId: row.id,
+ force,
+ },
+ })),
+ );
for (let i = 0; i < jobRows.length; i += CHILD_INSERT_CHUNK) {
const chunk = jobRows.slice(i, i + CHILD_INSERT_CHUNK);
- await db.insert(jobs).values(chunk);
+ await db.insert(jobs).values(chunk.map(prepareJob)).onConflictDoNothing();
}
dispatchedCount += results.length;
@@ -247,23 +234,8 @@ export async function processBulkTaggingDispatchJob(job: Job): Promise {
}
const jobRepo = services.getJobRepository();
- const parentJob = await jobRepo.findById(parentId);
- const parentPayload = batchParentPayloadSchema.parse(
- parentJob?.payload ?? {},
- );
- const [{ count: rawTotalChildCount }] = await db
- .select({ count: sql`count(*)` })
- .from(jobs)
- .where(and(eq(jobs.parentId, parentId), eq(jobs.type, "auto_tagging")));
- const totalChildCount = Number(rawTotalChildCount ?? 0);
- const progress = {
- processed: parentPayload.processed,
- failed: parentPayload.failed,
- total: totalChildCount,
- };
- await jobRepo.update(parentId, {
- payload: { ...parentPayload, total: totalChildCount },
- });
+ const progress = await jobRepo.recomputeBatchProgress(parentId);
+ const totalChildCount = progress?.total ?? 0;
if (totalChildCount === 0) {
await jobRepo.update(parentId, { status: "completed" });
@@ -274,12 +246,9 @@ export async function processBulkTaggingDispatchJob(job: Job): Promise {
} else {
RealtimeEventBus.publishJob("job-progress", {
jobId: parentId,
- processed: progress.processed,
+ processed: progress?.processed ?? 0,
total: totalChildCount,
});
- if (progress.processed + progress.failed >= progress.total) {
- await finalizeBatchParent(parentId, progress);
- }
}
logger.info(
diff --git a/apps/server/src/infrastructure/jobs/thumbnails.ts b/apps/server/src/infrastructure/jobs/thumbnails.ts
index 96af4ceeb..810e89ce5 100644
--- a/apps/server/src/infrastructure/jobs/thumbnails.ts
+++ b/apps/server/src/infrastructure/jobs/thumbnails.ts
@@ -1,12 +1,15 @@
import fs from "node:fs/promises";
import path from "node:path";
+import { prepareJob } from "@solid-imager/core/domain/jobs/registry";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
+import type { NewJob } from "@solid-imager/core/domain/repositories/job-repository";
import { services } from "~/application/registry";
// import {
// selectMediaById,
// selectMediaBySourceId,
// } from "~/infrastructure/db/queries/media"; // Removed
import { db } from "~/infrastructure/db";
-import { jobs, type Media, type NewJob } from "~/infrastructure/db/schema";
+import { jobs, type Media } from "~/infrastructure/db/schema";
import { ImageProcessor } from "~/infrastructure/processing/image-processor";
import { MediaRepository } from "~/infrastructure/repositories/media-repository"; // Added
import { DrizzleSourceRepository } from "~/infrastructure/repositories/source-repository";
@@ -147,20 +150,31 @@ export async function generateThumbnailsForSource(
// Use processMedia job type for unified processing
const basePath = (mediaSource.connectionInfo as { path: string }).path;
- const jobRows: NewJob[] = mediaItems.map((media) => ({
- type: "processMedia",
- mediaSourceId,
- payload: {
- mediaId: media.id,
- sourcePath: basePath,
+ const jobRows: NewJob[] = await Promise.all(
+ mediaItems.map(async (media) => ({
type: "processMedia",
- },
- }));
+ mediaSourceId,
+ targetId: media.id,
+ inputRevision: await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ }),
+ payload: {
+ mediaId: media.id,
+ sourcePath: basePath,
+ type: "processMedia",
+ },
+ })),
+ );
if (jobRows.length === 0) return 0;
const BATCH_SIZE = 500;
for (let i = 0; i < jobRows.length; i += BATCH_SIZE) {
const chunk = jobRows.slice(i, i + BATCH_SIZE);
- await db.insert(jobs).values(chunk);
+ await db.insert(jobs).values(chunk.map(prepareJob)).onConflictDoNothing();
}
// Jobs start automatically via worker
diff --git a/apps/server/src/infrastructure/processing/media-region-renderer.ts b/apps/server/src/infrastructure/processing/media-region-renderer.ts
new file mode 100644
index 000000000..96aa8282b
--- /dev/null
+++ b/apps/server/src/infrastructure/processing/media-region-renderer.ts
@@ -0,0 +1,106 @@
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type {
+ IMediaRegionRenderer,
+ RenderedMediaRegion,
+} from "@solid-imager/application/ports/media-region-service";
+import type { Media } from "@solid-imager/core/domain/media/schemas";
+import type {
+ MediaRegion,
+ MediaRegionRenderProfile,
+} from "@solid-imager/core/domain/media-regions/schemas";
+import type { SourceRepository } from "@solid-imager/core/domain/repositories/source-repository";
+import { localConnectionSchema } from "@solid-imager/core/domain/sources/schemas";
+import sharp from "sharp";
+
+function resolveSafePath(basePath: string, targetPath: string): string {
+ const absoluteBase = path.resolve(basePath);
+ const resolved = path.resolve(absoluteBase, targetPath);
+ if (
+ resolved !== absoluteBase &&
+ !resolved.startsWith(`${absoluteBase}${path.sep}`)
+ ) {
+ throw new Error("Media path escapes its configured source.");
+ }
+ return resolved;
+}
+
+function getExtraction(media: Media, region: MediaRegion) {
+ if (
+ region.x === null ||
+ region.y === null ||
+ region.width === null ||
+ region.height === null
+ ) {
+ throw new Error("Region does not contain crop bounds.");
+ }
+ const left = Math.max(0, Math.floor(region.x * media.width));
+ const top = Math.max(0, Math.floor(region.y * media.height));
+ const right = Math.min(
+ media.width,
+ Math.ceil((region.x + region.width) * media.width),
+ );
+ const bottom = Math.min(
+ media.height,
+ Math.ceil((region.y + region.height) * media.height),
+ );
+ return {
+ left,
+ top,
+ width: Math.max(1, right - left),
+ height: Math.max(1, bottom - top),
+ };
+}
+
+export class SharpMediaRegionRenderer implements IMediaRegionRenderer {
+ readonly version = "sharp-webp-isnetis-v1";
+
+ constructor(private readonly sourceRepository: SourceRepository) {}
+
+ async render(
+ media: Media,
+ region: MediaRegion,
+ profile: MediaRegionRenderProfile,
+ ): Promise {
+ const source = await this.sourceRepository.findById(media.mediaSourceId);
+ if (source?.type !== "local") {
+ throw new Error("Only local media sources support region rendering.");
+ }
+ const connection = localConnectionSchema.parse(source.connectionInfo);
+ const sourcePath = resolveSafePath(connection.path, media.filePath);
+ const extraction = getExtraction(media, region);
+
+ if (!profile.transparent) {
+ const bytes = await sharp(sourcePath)
+ .extract(extraction)
+ .webp()
+ .toBuffer();
+ return {
+ bytes,
+ format: "webp",
+ width: extraction.width,
+ height: extraction.height,
+ };
+ }
+
+ const temporaryPath = path.join(
+ tmpdir(),
+ `solid-imager-region-${crypto.randomUUID()}.png`,
+ );
+ await sharp(sourcePath).extract(extraction).png().toFile(temporaryPath);
+ try {
+ const { segmentRgbaWithIsnetis } = await import("dghs-imgutils-rs");
+ const bytes = new Uint8Array(await segmentRgbaWithIsnetis(temporaryPath));
+ return {
+ bytes,
+ format: "png",
+ width: extraction.width,
+ height: extraction.height,
+ };
+ } finally {
+ await Bun.file(temporaryPath)
+ .delete()
+ .catch(() => undefined);
+ }
+ }
+}
diff --git a/apps/server/src/infrastructure/repositories/media-region-repository.ts b/apps/server/src/infrastructure/repositories/media-region-repository.ts
new file mode 100644
index 000000000..18a6a300e
--- /dev/null
+++ b/apps/server/src/infrastructure/repositories/media-region-repository.ts
@@ -0,0 +1,6 @@
+import type { IMediaRegionRepository } from "@solid-imager/core/domain/repositories/media-region-repository";
+import { createMediaRegionRepository } from "@solid-imager/db/repositories/media-region-repository";
+import { getExecutor } from "~/infrastructure/db/executor";
+
+export const DrizzleMediaRegionRepository: IMediaRegionRepository =
+ createMediaRegionRepository(getExecutor);
diff --git a/apps/server/src/routes/api/media-regions.$regionId.render.ts b/apps/server/src/routes/api/media-regions.$regionId.render.ts
new file mode 100644
index 000000000..0047e3eee
--- /dev/null
+++ b/apps/server/src/routes/api/media-regions.$regionId.render.ts
@@ -0,0 +1,23 @@
+import { createFileRoute } from "@tanstack/solid-router";
+import { services } from "~/application/registry";
+import { handleMediaRegionRenderRequest } from "~/infrastructure/api/media-region-render-handler";
+import type { ServerRouteContext } from "~/infrastructure/router/route-types";
+import { bootstrapServerRoute } from "~/infrastructure/server-route-bootstrap";
+
+export const Route = createFileRoute("/api/media-regions/$regionId/render")({
+ server: {
+ handlers: {
+ GET: async ({
+ params,
+ request,
+ }: ServerRouteContext<{ regionId: string }>) => {
+ bootstrapServerRoute();
+ return handleMediaRegionRenderRequest(
+ request,
+ params.regionId,
+ services.getMediaRegionService(),
+ );
+ },
+ },
+ },
+});
diff --git a/apps/server/src/tests/integration/ai/ccip-revision-parity.test.ts b/apps/server/src/tests/integration/ai/ccip-revision-parity.test.ts
new file mode 100644
index 000000000..d80166b35
--- /dev/null
+++ b/apps/server/src/tests/integration/ai/ccip-revision-parity.test.ts
@@ -0,0 +1,118 @@
+import {
+ createCcipEmbeddingInputRevision,
+ createMediaRegionRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
+import { afterEach, describe, expect, it } from "vite-plus/test";
+import { createPglite } from "~/infrastructure/db/pglite";
+
+const MEDIA_ID = "11111111-1111-4111-8111-111111111111";
+const SOURCE_ID = "22222222-2222-4222-8222-222222222222";
+const MODIFIED_AT = new Date("2026-07-23T01:02:03.456Z");
+
+describe("CCIP revision SQL parity", () => {
+ let client: ReturnType | undefined;
+
+ afterEach(async () => {
+ await client?.close();
+ client = undefined;
+ });
+
+ it("matches the TypeScript source and full-region SHA-256 payloads", async () => {
+ client = createPglite();
+ const sourceRevision = await createMediaSourceRevision({
+ mediaId: MEDIA_ID,
+ mediaSourceId: SOURCE_ID,
+ modifiedAt: MODIFIED_AT,
+ fileSize: 123_456,
+ width: 1024,
+ height: 768,
+ });
+ const sourceResult = await client.query<{ revision: string }>(`
+ SELECT encode(
+ sha256(
+ convert_to(
+ concat(
+ '{"version":1,"mediaId":', to_json($1::text)::text,
+ ',"mediaSourceId":', to_json($2::text)::text,
+ ',"modifiedAtMs":', floor(extract(epoch FROM $3::timestamp) * 1000)::bigint,
+ ',"fileSize":', $4::bigint,
+ ',"width":', $5::integer,
+ ',"height":', $6::integer, '}'
+ ),
+ 'UTF8'
+ )
+ ),
+ 'hex'
+ ) AS revision
+ `, [
+ MEDIA_ID,
+ SOURCE_ID,
+ MODIFIED_AT,
+ 123_456,
+ 1024,
+ 768,
+ ]);
+ expect(sourceResult.rows[0]?.revision).toBe(sourceRevision);
+ const inputRevision = await createCcipEmbeddingInputRevision({
+ sourceRevision,
+ model: "ccip-model",
+ embeddingVersion: 1,
+ preprocessingProfile: "dghs-imgutils-rs/full-image-default/v1",
+ });
+ const inputResult = await client.query<{ revision: string }>(`
+ SELECT encode(
+ sha256(
+ convert_to(
+ concat(
+ '{"version":1,"sourceRevision":', to_json($1::text)::text,
+ ',"model":', to_json($2::text)::text,
+ ',"embeddingVersion":', $3::integer,
+ ',"preprocessingProfile":', to_json($4::text)::text, '}'
+ ),
+ 'UTF8'
+ )
+ ),
+ 'hex'
+ ) AS revision
+ `, [
+ sourceRevision,
+ "ccip-model",
+ 1,
+ "dghs-imgutils-rs/full-image-default/v1",
+ ]);
+ expect(inputResult.rows[0]?.revision).toBe(inputRevision);
+
+ const regionRevision = await createMediaRegionRevision({
+ sourceRevision,
+ kind: "full",
+ x: null,
+ y: null,
+ width: null,
+ height: null,
+ label: null,
+ detector: null,
+ detectorModel: null,
+ detectorVersion: null,
+ manualReason: null,
+ });
+ const regionResult = await client.query<{ revision: string }>(`
+ SELECT encode(
+ sha256(
+ convert_to(
+ concat(
+ '{"version":1,"sourceRevision":', to_json($1::text)::text,
+ ',"kind":', to_json('full'::text)::text,
+ ',"x":null,"y":null,"width":null,"height":null',
+ ',"label":null,"detector":null,"detectorModel":null',
+ ',"detectorVersion":null,"manualReason":null}'
+ ),
+ 'UTF8'
+ )
+ ),
+ 'hex'
+ ) AS revision
+ `, [sourceRevision]);
+ expect(regionResult.rows[0]?.revision).toBe(regionRevision);
+ });
+});
diff --git a/apps/server/src/tests/integration/ai/lancedb-ccip-vector-store.test.ts b/apps/server/src/tests/integration/ai/lancedb-ccip-vector-store.test.ts
index 83cd42b60..a9081b3f0 100644
--- a/apps/server/src/tests/integration/ai/lancedb-ccip-vector-store.test.ts
+++ b/apps/server/src/tests/integration/ai/lancedb-ccip-vector-store.test.ts
@@ -6,8 +6,13 @@ import type { ITaggingService } from "@solid-imager/application/ports/tagging-se
import {
CCIP_EMBEDDING_VERSION,
CCIP_MODEL,
+ CCIP_PREPROCESSING_PROFILE,
CcipVectorService,
} from "@solid-imager/application/services/ccip-vector-service";
+import {
+ createCcipEmbeddingInputRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
import type { Media } from "@solid-imager/core/domain/media/schemas";
import type { IMediaRepository } from "@solid-imager/core/domain/repositories/media-repository";
import type {
@@ -15,7 +20,10 @@ import type {
SourceRepository,
} from "@solid-imager/core/domain/repositories/source-repository";
import { afterEach, describe, expect, it } from "vite-plus/test";
-import { LanceDbCcipVectorStore } from "~/infrastructure/ai/lancedb-ccip-vector-store";
+import {
+ CcipStoreUnsupportedOperationError,
+ LanceDbCcipVectorStore,
+} from "~/infrastructure/ai/lancedb-ccip-vector-store";
const SOURCE_A_ID = "11111111-1111-4111-8111-111111111111";
const SOURCE_B_ID = "22222222-2222-4222-8222-222222222222";
@@ -29,8 +37,37 @@ const EXTRACTED_AT = new Date("2026-07-02T00:00:00.000Z");
const READ_QUERY = {
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
};
+const revisionByMediaId = new Map(
+ await Promise.all(
+ [
+ [ANCHOR_MEDIA_ID, SOURCE_A_ID],
+ [NEAR_MEDIA_ID, SOURCE_A_ID],
+ [FAR_MEDIA_ID, SOURCE_A_ID],
+ [OTHER_SOURCE_MEDIA_ID, SOURCE_B_ID],
+ ].map(async ([mediaId, mediaSourceId]): Promise<[string, string]> => {
+ return [
+ mediaId,
+ await createCcipEmbeddingInputRevision({
+ sourceRevision: await createMediaSourceRevision({
+ mediaId,
+ mediaSourceId,
+ modifiedAt: MODIFIED_AT,
+ fileSize: 1,
+ width: 256,
+ height: 256,
+ }),
+ model: CCIP_MODEL,
+ embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ }),
+ ];
+ }),
+ ),
+);
+
function vector(first: number, second = 0): number[] {
return Array.from({ length: 768 }, (_, index) => {
if (index === 0) return first;
@@ -74,13 +111,19 @@ function createRecord(
mediaSourceId: string,
feature: number[],
): CcipVectorRecord {
+ const inputRevision = revisionByMediaId.get(mediaId);
+ if (!inputRevision) throw new Error(`Missing revision fixture: ${mediaId}`);
return {
+ regionId: mediaId,
+ regionKind: "full",
mediaId,
mediaSourceId,
vector: feature,
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
mediaModifiedAt: MODIFIED_AT,
+ inputRevision,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
extractedAt: EXTRACTED_AT,
};
}
@@ -135,10 +178,14 @@ describe("LanceDbCcipVectorStore integration", () => {
const metadata = await store.getMetadataMany([ANCHOR_MEDIA_ID], READ_QUERY);
expect(metadata.get(ANCHOR_MEDIA_ID)).toEqual({
mediaId: anchor.mediaId,
+ regionId: anchor.regionId,
+ regionKind: anchor.regionKind,
mediaSourceId: anchor.mediaSourceId,
model: anchor.model,
embeddingVersion: anchor.embeddingVersion,
mediaModifiedAt: anchor.mediaModifiedAt,
+ inputRevision: anchor.inputRevision,
+ preprocessingProfile: anchor.preprocessingProfile,
extractedAt: anchor.extractedAt,
});
@@ -270,4 +317,61 @@ describe("LanceDbCcipVectorStore integration", () => {
FAR_MEDIA_ID,
]);
});
+
+ it("keeps extraction, currentness checks, and deletion compatible with the legacy schema", async () => {
+ directory = await mkdtemp(
+ path.join(tmpdir(), "solid-imager-ccip-lancedb-legacy-"),
+ );
+ const vectorStore = new LanceDbCcipVectorStore(directory, { legacy: true });
+ const media = createMedia(ANCHOR_MEDIA_ID, SOURCE_A_ID);
+ const source = createSource(SOURCE_A_ID);
+ let extractionCalls = 0;
+ const taggingService: ITaggingService = {
+ ...createTaggingService(),
+ getCcipFeatureForMedia: async () => {
+ extractionCalls += 1;
+ return { feature: vector(1) };
+ },
+ };
+ const service = new CcipVectorService({
+ mediaRepository: {
+ findById: async (id: string) => (id === media.id ? media : null),
+ } as IMediaRepository,
+ sourceRepository: {
+ findById: async (id: string) => (id === source.id ? source : null),
+ } as SourceRepository,
+ taggingService,
+ vectorStore,
+ });
+
+ const extracted = await service.extract(source.id, media.id);
+ expect(extracted.skipped).toBe(false);
+ expect(extractionCalls).toBe(1);
+ const stored = await vectorStore.get(media.id, READ_QUERY);
+ expect(stored).toMatchObject({
+ regionId: null,
+ regionKind: "full",
+ mediaId: media.id,
+ inputRevision: "legacy-unversioned",
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ });
+
+ const second = await service.extract(source.id, media.id);
+ expect(second.skipped).toBe(true);
+ expect(extractionCalls).toBe(1);
+ expect(await service.getStatus(source.id, media.id)).toMatchObject({
+ status: "ready",
+ });
+ await expect(
+ vectorStore.deleteEmbedding({
+ regionId: media.id,
+ model: CCIP_MODEL,
+ embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ }),
+ ).rejects.toBeInstanceOf(CcipStoreUnsupportedOperationError);
+
+ await service.delete(media.id);
+ expect(await vectorStore.get(media.id, READ_QUERY)).toBeNull();
+ });
});
diff --git a/apps/server/src/tests/integration/ai/postgres-ccip-vector-store.test.ts b/apps/server/src/tests/integration/ai/postgres-ccip-vector-store.test.ts
index 340a94488..cf85a7f26 100644
--- a/apps/server/src/tests/integration/ai/postgres-ccip-vector-store.test.ts
+++ b/apps/server/src/tests/integration/ai/postgres-ccip-vector-store.test.ts
@@ -3,7 +3,12 @@ import type { CcipVectorRecord } from "@solid-imager/application/ports/ccip-vect
import {
CCIP_EMBEDDING_VERSION,
CCIP_MODEL,
+ CCIP_PREPROCESSING_PROFILE,
} from "@solid-imager/application/services/ccip-vector-service";
+import {
+ createCcipEmbeddingInputRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
import {
CCIP_VECTOR_DIMENSIONS,
mediaRegions,
@@ -30,6 +35,21 @@ const EXTRACTED_AT = new Date("2026-07-02T00:00:00.000Z");
const NEWER_MODIFIED_AT = new Date("2026-07-03T00:00:00.000Z");
const NEWER_EXTRACTED_AT = new Date("2026-07-04T00:00:00.000Z");
+const sourceARevision = await createMediaSourceRevision({
+ mediaId: ANCHOR_MEDIA_ID,
+ mediaSourceId: SOURCE_A_ID,
+ modifiedAt: MODIFIED_AT,
+ fileSize: null,
+ width: 256,
+ height: 256,
+});
+const sourceAInputRevision = await createCcipEmbeddingInputRevision({
+ sourceRevision: sourceARevision,
+ model: CCIP_MODEL,
+ embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+});
+
function vector(first: number, second = 0): number[] {
return Array.from({ length: CCIP_VECTOR_DIMENSIONS }, (_, index) => {
if (index === 0) return first;
@@ -44,16 +64,46 @@ function record(
feature: number[],
): CcipVectorRecord {
return {
+ regionId: null,
+ regionKind: "full",
mediaId,
mediaSourceId,
vector: feature,
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
mediaModifiedAt: MODIFIED_AT,
+ inputRevision:
+ mediaId === ANCHOR_MEDIA_ID
+ ? sourceAInputRevision
+ : "set-by-test-before-write",
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
extractedAt: EXTRACTED_AT,
};
}
+async function currentRecord(
+ mediaId: string,
+ mediaSourceId: string,
+ feature: number[],
+): Promise {
+ return {
+ ...record(mediaId, mediaSourceId, feature),
+ inputRevision: await createCcipEmbeddingInputRevision({
+ sourceRevision: await createMediaSourceRevision({
+ mediaId,
+ mediaSourceId,
+ modifiedAt: MODIFIED_AT,
+ fileSize: null,
+ width: 256,
+ height: 256,
+ }),
+ model: CCIP_MODEL,
+ embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ }),
+ };
+}
+
describe("PostgresCcipVectorStore", () => {
let client: ReturnType | undefined;
@@ -104,49 +154,58 @@ describe("PostgresCcipVectorStore", () => {
const store = new PostgresCcipVectorStore(database);
const anchor = record(ANCHOR_MEDIA_ID, SOURCE_A_ID, vector(1));
- await store.upsertMany([
- anchor,
- {
- ...record(ANCHOR_MEDIA_ID, SOURCE_A_ID, vector(0, 1)),
+ const legacyAnchor = {
+ ...record(ANCHOR_MEDIA_ID, SOURCE_A_ID, vector(0, 1)),
+ model: "ccip-legacy-model",
+ embeddingVersion: 2,
+ inputRevision: await createCcipEmbeddingInputRevision({
+ sourceRevision: sourceARevision,
model: "ccip-legacy-model",
embeddingVersion: 2,
- },
- record(NEAR_MEDIA_ID, SOURCE_A_ID, vector(0.875, 0.125)),
- record(FAR_MEDIA_ID, SOURCE_A_ID, vector(0, 1)),
- record(OTHER_SOURCE_MEDIA_ID, SOURCE_B_ID, vector(0.75, 0.25)),
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ }),
+ };
+ const records = await Promise.all([
+ currentRecord(NEAR_MEDIA_ID, SOURCE_A_ID, vector(0.875, 0.125)),
+ currentRecord(FAR_MEDIA_ID, SOURCE_A_ID, vector(0, 1)),
+ currentRecord(
+ OTHER_SOURCE_MEDIA_ID,
+ SOURCE_B_ID,
+ vector(0.75, 0.25),
+ ),
+ ]);
+ await store.upsertMany([
+ anchor,
+ legacyAnchor,
+ ...records,
]);
- expect(
- await store.get(ANCHOR_MEDIA_ID, {
+ const storedAnchor = await store.get(ANCHOR_MEDIA_ID, {
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
- }),
- ).toEqual(anchor);
- expect(
- await store.getMetadataMany([ANCHOR_MEDIA_ID], {
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ });
+ expect(storedAnchor).toEqual({
+ ...anchor,
+ regionId: expect.any(String),
+ });
+ const metadata = await store.getMetadataMany([ANCHOR_MEDIA_ID], {
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
- }),
- ).toEqual(
- new Map([
- [
- ANCHOR_MEDIA_ID,
- {
- mediaId: ANCHOR_MEDIA_ID,
- mediaSourceId: SOURCE_A_ID,
- model: CCIP_MODEL,
- embeddingVersion: CCIP_EMBEDDING_VERSION,
- mediaModifiedAt: MODIFIED_AT,
- extractedAt: EXTRACTED_AT,
- },
- ],
- ]),
- );
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ });
+ const { vector: omittedVector, ...anchorMetadata } = anchor;
+ void omittedVector;
+ expect(metadata.get(ANCHOR_MEDIA_ID)).toEqual({
+ ...anchorMetadata,
+ regionId: expect.any(String),
+ });
const candidates = await store.search(anchor.vector, 10, {
mediaSourceId: SOURCE_A_ID,
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
});
expect(candidates.map((candidate) => candidate.mediaId)).toEqual([
ANCHOR_MEDIA_ID,
@@ -157,20 +216,40 @@ describe("PostgresCcipVectorStore", () => {
OTHER_SOURCE_MEDIA_ID,
);
+ await database
+ .update(medias)
+ .set({ modifiedAt: NEWER_MODIFIED_AT })
+ .where(eq(medias.id, ANCHOR_MEDIA_ID));
const newerAnchor = {
...anchor,
vector: vector(0.5, 0.5),
mediaModifiedAt: NEWER_MODIFIED_AT,
+ inputRevision: await createCcipEmbeddingInputRevision({
+ sourceRevision: await createMediaSourceRevision({
+ mediaId: ANCHOR_MEDIA_ID,
+ mediaSourceId: SOURCE_A_ID,
+ modifiedAt: NEWER_MODIFIED_AT,
+ fileSize: null,
+ width: 256,
+ height: 256,
+ }),
+ model: CCIP_MODEL,
+ embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ }),
extractedAt: NEWER_EXTRACTED_AT,
};
await store.upsert(newerAnchor);
- await store.upsert(anchor);
+ await expect(store.upsert(anchor)).rejects.toThrow(
+ "CCIP input revision changed before commit",
+ );
expect(
await store.get(ANCHOR_MEDIA_ID, {
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
}),
- ).toEqual(newerAnchor);
+ ).toEqual({ ...newerAnchor, regionId: expect.any(String) });
const [region] = await database
.select({ sourceModifiedAt: mediaRegions.sourceModifiedAt })
.from(mediaRegions)
diff --git a/apps/server/src/tests/integration/repository/job-repository.test.ts b/apps/server/src/tests/integration/repository/job-repository.test.ts
new file mode 100644
index 000000000..a54e262bb
--- /dev/null
+++ b/apps/server/src/tests/integration/repository/job-repository.test.ts
@@ -0,0 +1,249 @@
+import path from 'node:path';
+import { createJobRepository } from '@solid-imager/db/repositories/job-repository';
+import { jobs, lanceDbSyncDirty, mediaSources } from '@solid-imager/db/schema';
+import { and, desc, eq } from 'drizzle-orm';
+import { drizzle } from 'drizzle-orm/pglite';
+import { migrate } from 'drizzle-orm/pglite/migrator';
+import { afterEach, beforeEach, describe, expect, it } from 'vite-plus/test';
+import { createPglite } from '~/infrastructure/db/pglite';
+import { ccipJobTargetsMedia } from '~/infrastructure/jobs/ccip-job-query';
+import * as schema from '~/infrastructure/db/schema';
+
+const SOURCE_ID = '10000000-0000-4000-8000-000000000001';
+const MEDIA_A = '20000000-0000-4000-8000-000000000001';
+const MEDIA_B = '20000000-0000-4000-8000-000000000002';
+const MEDIA_C = '20000000-0000-4000-8000-000000000003';
+
+describe('JobRepository durable claims', () => {
+ let client: ReturnType | undefined;
+ let database: ReturnType>;
+
+ beforeEach(async () => {
+ client = createPglite();
+ database = drizzle(client, { schema });
+ const migrationsFolder = process.cwd().endsWith('apps/server')
+ ? path.resolve(process.cwd(), 'drizzle')
+ : path.resolve(process.cwd(), 'apps/server/drizzle');
+ await migrate(database, { migrationsFolder });
+ await database.insert(mediaSources).values({
+ id: SOURCE_ID,
+ name: 'Job source',
+ description: null,
+ type: 'local',
+ connectionInfo: { path: '/tmp/job-source' },
+ });
+ });
+
+ afterEach(async () => {
+ await client?.close();
+ client = undefined;
+ });
+
+ it('stores delta changes in the dirty table and creates a follow-up for running work', async () => {
+ const repository = createJobRepository(() => database);
+ await repository.createIfUnique({
+ type: 'sync_lancedb_delta',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaIds: [MEDIA_A] },
+ });
+ await Promise.all([
+ repository.createIfUnique({
+ type: 'sync_lancedb_delta',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaIds: [MEDIA_B] },
+ }),
+ repository.createIfUnique({
+ type: 'sync_lancedb_delta',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaIds: [MEDIA_C] },
+ }),
+ ]);
+
+ const [pending] = await database
+ .select()
+ .from(jobs)
+ .where(eq(jobs.status, 'pending'));
+ expect(pending.payload).toEqual({ reason: 'dirty' });
+ const dirtyRows = await database.select().from(lanceDbSyncDirty);
+ expect(dirtyRows).toEqual(
+ expect.arrayContaining(
+ [MEDIA_A, MEDIA_B, MEDIA_C].map((mediaId) =>
+ expect.objectContaining({ mediaId, operation: 'upsert' }),
+ ),
+ ),
+ );
+
+ const [claimed] = await repository.claimPending(1, {
+ includeTypes: ['sync_lancedb_delta'],
+ queueNames: ['default'],
+ workerId: 'integration-worker',
+ });
+ expect(claimed.claimToken).toBeTruthy();
+ await repository.createIfUnique({
+ type: 'sync_lancedb_delta',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaIds: [MEDIA_A] },
+ });
+ const activeRows = await database
+ .select()
+ .from(jobs)
+ .where(eq(jobs.type, 'sync_lancedb_delta'));
+ expect(activeRows.map((row) => row.status).sort()).toEqual([
+ 'in_progress',
+ 'pending',
+ ]);
+ expect(activeRows.some((row) => row.dedupeKey?.endsWith(':followup'))).toBe(
+ true,
+ );
+ });
+
+ it('preserves mixed upsert and delete operations per dirty media row', async () => {
+ const repository = createJobRepository(() => database);
+ await repository.createIfUnique({
+ type: 'sync_lancedb_delta',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaIds: [MEDIA_A], operation: 'upsert' },
+ });
+ await repository.createIfUnique({
+ type: 'sync_lancedb_delta',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaIds: [MEDIA_B], operation: 'delete' },
+ });
+
+ let dirtyRows = await database
+ .select()
+ .from(lanceDbSyncDirty)
+ .where(eq(lanceDbSyncDirty.mediaSourceId, SOURCE_ID));
+ expect(
+ dirtyRows.map(({ mediaId, operation }) => ({ mediaId, operation })),
+ ).toEqual(
+ expect.arrayContaining([
+ { mediaId: MEDIA_A, operation: 'upsert' },
+ { mediaId: MEDIA_B, operation: 'delete' },
+ ]),
+ );
+
+ await repository.createIfUnique({
+ type: 'sync_lancedb_delta',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaId: MEDIA_A, operation: 'delete' },
+ });
+ dirtyRows = await database
+ .select()
+ .from(lanceDbSyncDirty)
+ .where(eq(lanceDbSyncDirty.mediaId, MEDIA_A));
+ expect(dirtyRows[0]).toEqual(
+ expect.objectContaining({
+ mediaId: MEDIA_A,
+ operation: 'delete',
+ generation: 1,
+ }),
+ );
+ });
+
+ it('finds legacy CCIP batch jobs by a mediaIds payload member', async () => {
+ const repository = createJobRepository(() => database);
+ const legacyJob = await repository.create({
+ type: 'extract_ccip_vector',
+ mediaSourceId: SOURCE_ID,
+ payload: { mediaIds: [MEDIA_A, MEDIA_B], force: false },
+ });
+ await database
+ .update(jobs)
+ .set({ status: 'failed', errorCode: 'JOB_EXECUTION_FAILED' })
+ .where(eq(jobs.id, legacyJob.id));
+
+ const [latestJob] = await database
+ .select()
+ .from(jobs)
+ .where(
+ and(
+ eq(jobs.type, 'extract_ccip_vector'),
+ eq(jobs.mediaSourceId, SOURCE_ID),
+ ccipJobTargetsMedia(MEDIA_B),
+ ),
+ )
+ .orderBy(desc(jobs.createdAt))
+ .limit(1);
+
+ expect(latestJob).toEqual(
+ expect.objectContaining({
+ id: legacyJob.id,
+ status: 'failed',
+ errorCode: 'JOB_EXECUTION_FAILED',
+ }),
+ );
+ });
+
+ it('fences completion by token and input revision', async () => {
+ const repository = createJobRepository(() => database);
+ await repository.create({
+ type: 'processMedia',
+ mediaSourceId: SOURCE_ID,
+ targetId: MEDIA_A,
+ inputRevision: 'revision-a',
+ payload: { mediaId: MEDIA_A, sourcePath: '/tmp/job-source' },
+ });
+ const [claimed] = await repository.claimPending(1, {
+ includeTypes: ['processMedia'],
+ workerId: 'integration-worker',
+ });
+ expect(claimed.claimToken).toBeTruthy();
+ const wrongFence = await repository.completeClaim(claimed.id, {
+ claimToken: '30000000-0000-4000-8000-000000000001',
+ inputRevision: claimed.inputRevision,
+ });
+ expect(wrongFence).toBe(false);
+ const accepted = await repository.completeClaim(
+ claimed.id,
+ {
+ claimToken: claimed.claimToken ?? '',
+ inputRevision: claimed.inputRevision,
+ },
+ { success: true },
+ );
+ expect(accepted).toBe(true);
+ expect((await repository.findById(claimed.id))?.status).toBe('completed');
+ });
+
+ it('claims at most one pending row for each concurrency key', async () => {
+ const repository = createJobRepository(() => database);
+ for (const [mediaId, revision] of [
+ [MEDIA_A, 'revision-a'],
+ [MEDIA_B, 'revision-b'],
+ ] as const) {
+ await repository.create({
+ type: 'processMedia',
+ mediaSourceId: SOURCE_ID,
+ targetId: mediaId,
+ inputRevision: revision,
+ concurrencyKey: 'shared-test-key',
+ payload: { mediaId, sourcePath: '/tmp/job-source' },
+ });
+ }
+ const claimed = await repository.claimPending(2, {
+ includeTypes: ['processMedia'],
+ workerId: 'integration-worker',
+ });
+ expect(claimed).toHaveLength(1);
+ });
+
+ it('rolls back the parent when dispatch creation fails', async () => {
+ const repository = createJobRepository(() => database);
+ await expect(
+ repository.createParentWithDispatch(
+ {
+ type: 'bulk_tagging_parent',
+ status: 'in_progress',
+ payload: { total: 0, processed: 0, failed: 0 },
+ },
+ { type: 'unknown_dispatch', payload: {} },
+ ),
+ ).rejects.toThrow('Unknown job type');
+ const parentRows = await database
+ .select()
+ .from(jobs)
+ .where(eq(jobs.type, 'bulk_tagging_parent'));
+ expect(parentRows).toHaveLength(0);
+ });
+});
diff --git a/apps/server/src/tests/integration/repository/media-region-repository.test.ts b/apps/server/src/tests/integration/repository/media-region-repository.test.ts
new file mode 100644
index 000000000..55794aaaf
--- /dev/null
+++ b/apps/server/src/tests/integration/repository/media-region-repository.test.ts
@@ -0,0 +1,164 @@
+import path from "node:path";
+import {
+ createMediaRegionRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
+import { createMediaRegionRepository } from "@solid-imager/db/repositories/media-region-repository";
+import {
+ mediaRelationsTable,
+ mediaSources,
+ medias,
+} from "@solid-imager/db/schema";
+import { eq } from "drizzle-orm";
+import { drizzle } from "drizzle-orm/pglite";
+import { migrate } from "drizzle-orm/pglite/migrator";
+import { afterEach, describe, expect, it } from "vite-plus/test";
+import { createPglite } from "~/infrastructure/db/pglite";
+import * as schema from "~/infrastructure/db/schema";
+
+const SOURCE_ID = "10000000-0000-4000-8000-000000000001";
+const MEDIA_ID = "20000000-0000-4000-8000-000000000002";
+const MODIFIED_AT = new Date("2026-07-20T00:00:00.000Z");
+
+describe("MediaRegionRepository", () => {
+ let client: ReturnType | undefined;
+
+ afterEach(async () => {
+ await client?.close();
+ client = undefined;
+ });
+
+ it("uses optimistic revisions and preserves a derivative after region deletion", async () => {
+ client = createPglite();
+ const database = drizzle(client, { schema });
+ const migrationsFolder = process.cwd().endsWith("apps/server")
+ ? path.resolve(process.cwd(), "drizzle")
+ : path.resolve(process.cwd(), "apps/server/drizzle");
+ await migrate(database, { migrationsFolder });
+ await database.insert(mediaSources).values({
+ id: SOURCE_ID,
+ name: "Region source",
+ description: null,
+ type: "local",
+ connectionInfo: { path: "/tmp/region-source" },
+ });
+ await database.insert(medias).values({
+ id: MEDIA_ID,
+ mediaSourceId: SOURCE_ID,
+ filePath: "source.png",
+ fileName: "source.png",
+ mediaType: "image",
+ width: 100,
+ height: 200,
+ fileSize: 500,
+ modifiedAt: MODIFIED_AT,
+ });
+
+ const sourceRevision = await createMediaSourceRevision({
+ mediaId: MEDIA_ID,
+ mediaSourceId: SOURCE_ID,
+ modifiedAt: MODIFIED_AT,
+ fileSize: 500,
+ width: 100,
+ height: 200,
+ });
+ const bbox = { x: 0.1, y: 0.2, width: 0.3, height: 0.4 };
+ const initialRevision = await createMediaRegionRevision({
+ sourceRevision,
+ kind: "manual",
+ ...bbox,
+ label: "first",
+ detector: null,
+ detectorModel: null,
+ detectorVersion: null,
+ manualReason: "test",
+ });
+ const repository = createMediaRegionRepository(() => database);
+ const region = await repository.create({
+ mediaId: MEDIA_ID,
+ kind: "manual",
+ bbox,
+ sourceWidth: 100,
+ sourceHeight: 200,
+ sourceModifiedAt: MODIFIED_AT,
+ sourceRevision,
+ regionRevision: initialRevision,
+ label: "first",
+ manualReason: "test",
+ detectionKey: null,
+ detector: null,
+ detectorModel: null,
+ detectorVersion: null,
+ score: null,
+ });
+
+ const nextRevision = await createMediaRegionRevision({
+ sourceRevision,
+ kind: "manual",
+ ...bbox,
+ label: "updated",
+ detector: null,
+ detectorModel: null,
+ detectorVersion: null,
+ manualReason: "test",
+ });
+ const updated = await repository.update(region.id, initialRevision, {
+ label: "updated",
+ regionRevision: nextRevision,
+ updatedAt: new Date("2026-07-21T00:00:00.000Z"),
+ });
+ expect(updated?.label).toBe("updated");
+ await expect(
+ repository.update(region.id, initialRevision, {
+ label: "lost update",
+ regionRevision: "a".repeat(64),
+ updatedAt: new Date(),
+ }),
+ ).resolves.toBeNull();
+
+ const child = await repository.createMaterialized({
+ media: {
+ mediaSourceId: SOURCE_ID,
+ filePath: "source.region.webp",
+ fileName: "source.region.webp",
+ mediaType: "image",
+ width: 30,
+ height: 80,
+ fileSize: 200,
+ description: null,
+ createdAt: MODIFIED_AT,
+ modifiedAt: MODIFIED_AT,
+ },
+ parentMediaId: MEDIA_ID,
+ sourceRegionId: region.id,
+ derivationKey: "derivation-key",
+ snapshot: {
+ regionId: region.id,
+ regionRevision: nextRevision,
+ sourceRevision,
+ bbox,
+ label: "updated",
+ profile: { transparent: false },
+ profileVersion: "crop-v1",
+ rendererVersion: "test-renderer-v1",
+ },
+ });
+ expect(
+ await repository.findMaterializedByDerivationKey("derivation-key"),
+ ).toMatchObject({ id: child.id });
+
+ await expect(repository.delete(region.id, nextRevision)).resolves.toBe(
+ true,
+ );
+ const [relation] = await database
+ .select()
+ .from(mediaRelationsTable)
+ .where(eq(mediaRelationsTable.derivationKey, "derivation-key"));
+ expect(relation?.sourceRegionId).toBeNull();
+ const [persistedChild] = await database
+ .select()
+ .from(medias)
+ .where(eq(medias.id, child.id));
+ expect(persistedChild?.id).toBe(child.id);
+ });
+});
diff --git a/apps/server/src/tests/unit/application/services/ccip-vector-service.test.ts b/apps/server/src/tests/unit/application/services/ccip-vector-service.test.ts
index cee22c595..bfe2683af 100644
--- a/apps/server/src/tests/unit/application/services/ccip-vector-service.test.ts
+++ b/apps/server/src/tests/unit/application/services/ccip-vector-service.test.ts
@@ -1,4 +1,11 @@
-import { CcipVectorService } from "@solid-imager/application/services/ccip-vector-service";
+import {
+ CCIP_PREPROCESSING_PROFILE,
+ CcipVectorService,
+} from "@solid-imager/application/services/ccip-vector-service";
+import {
+ createCcipEmbeddingInputRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
import { describe, expect, it, vi } from "vitest";
const source = { id: "00000000-0000-4000-8000-000000000010", type: "local" };
@@ -7,17 +14,37 @@ const media = {
mediaSourceId: source.id,
mediaType: "image",
modifiedAt: new Date("2026-01-01T00:00:00Z"),
+ fileSize: 1,
+ width: 256,
+ height: 256,
};
+const inputRevision = await createCcipEmbeddingInputRevision({
+ sourceRevision: await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ }),
+ model: "ccip-caformer-24-randaug-pruned",
+ embeddingVersion: 1,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+});
describe("CcipVectorService", () => {
it("skips extraction when the stored vector is current", async () => {
const record = {
+ regionId: media.id,
+ regionKind: "full" as const,
mediaId: media.id,
mediaSourceId: source.id,
vector: Array.from({ length: 768 }, () => 0),
model: "ccip-caformer-24-randaug-pruned",
embeddingVersion: 1,
mediaModifiedAt: new Date(media.modifiedAt.getTime() - 500),
+ inputRevision,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
extractedAt: new Date(),
};
const taggingService = { getCcipFeatureForMedia: vi.fn() };
@@ -96,15 +123,40 @@ describe("CcipVectorService", () => {
...media,
id: "00000000-0000-4000-8000-000000000003",
};
- const record = (item: typeof media, vector: number[]) => ({
+ const record = async (item: typeof media, vector: number[]) => ({
+ regionId: item.id,
+ regionKind: "full" as const,
mediaId: item.id,
mediaSourceId: source.id,
vector,
model: "ccip-caformer-24-randaug-pruned",
embeddingVersion: 1,
mediaModifiedAt: item.modifiedAt,
+ inputRevision: await createCcipEmbeddingInputRevision({
+ sourceRevision: await createMediaSourceRevision({
+ mediaId: item.id,
+ mediaSourceId: item.mediaSourceId,
+ modifiedAt: item.modifiedAt,
+ fileSize: item.fileSize,
+ width: item.width,
+ height: item.height,
+ }),
+ model: "ccip-caformer-24-randaug-pruned",
+ embeddingVersion: 1,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ }),
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
extractedAt: new Date(),
});
+ const anchorRecord = await record(media, anchorVector);
+ const candidateARecord = await record(
+ candidateA,
+ Array.from({ length: 768 }, () => 1),
+ );
+ const candidateBRecord = await record(
+ candidateB,
+ Array.from({ length: 768 }, () => 2),
+ );
const service = new CcipVectorService({
mediaRepository: {
findById: vi.fn().mockResolvedValue(media),
@@ -117,20 +169,14 @@ describe("CcipVectorService", () => {
getCcipDistances: vi.fn().mockResolvedValue([0.4, 0.1]),
} as any,
vectorStore: {
- get: vi.fn().mockResolvedValue(record(media, anchorVector)),
+ get: vi.fn().mockResolvedValue(anchorRecord),
search: vi.fn().mockResolvedValue([
{
- ...record(
- candidateA,
- Array.from({ length: 768 }, () => 1),
- ),
+ ...candidateARecord,
cosineDistance: 0.1,
},
{
- ...record(
- candidateB,
- Array.from({ length: 768 }, () => 2),
- ),
+ ...candidateBRecord,
cosineDistance: 0.2,
},
]),
diff --git a/apps/server/src/tests/unit/application/services/maintenance-service.test.ts b/apps/server/src/tests/unit/application/services/maintenance-service.test.ts
index bb34179a4..575280e50 100644
--- a/apps/server/src/tests/unit/application/services/maintenance-service.test.ts
+++ b/apps/server/src/tests/unit/application/services/maintenance-service.test.ts
@@ -78,6 +78,7 @@ const mockMediaRepo = {
findIdsWithMissingGenerationInfo: vi.fn(),
findAllMediaIndices: vi.fn(),
findAllPathsBySourceId: vi.fn(),
+ findById: vi.fn(),
};
const mockJobRepo = {
@@ -97,7 +98,15 @@ function makeMedia(
mediaSourceId = "source-1",
filePath = `/media/${id}.png`,
) {
- return { id, mediaSourceId, filePath };
+ return {
+ id,
+ mediaSourceId,
+ filePath,
+ modifiedAt: new Date("2026-01-01T00:00:00.000Z"),
+ fileSize: 1024,
+ width: 100,
+ height: 100,
+ };
}
/** Build a minimal local media source record for test data. */
@@ -118,6 +127,9 @@ describe("MaintenanceService", () => {
);
mockSourceRepo.findAll.mockResolvedValue([]);
mockMediaRepo.findAllPathsBySourceId.mockResolvedValue([]);
+ mockMediaRepo.findById.mockImplementation((id: string) =>
+ Promise.resolve(makeMedia(id)),
+ );
});
afterEach(() => {
diff --git a/apps/server/src/tests/unit/application/services/media-service.test.ts b/apps/server/src/tests/unit/application/services/media-service.test.ts
index 4302fe4ea..1199a6930 100644
--- a/apps/server/src/tests/unit/application/services/media-service.test.ts
+++ b/apps/server/src/tests/unit/application/services/media-service.test.ts
@@ -105,9 +105,16 @@ describe("MediaService Unit Tests", () => {
mockJobRepository = {
create: vi.fn(),
createIfUnique: vi.fn(),
+ createParentWithDispatch: vi.fn(),
findById: vi.fn(),
findPending: vi.fn(),
claimPending: vi.fn(),
+ heartbeatClaim: vi.fn(),
+ completeClaim: vi.fn(),
+ failClaim: vi.fn(),
+ releaseClaim: vi.fn(),
+ recomputeBatchProgress: vi.fn(),
+ requeueExpiredLeases: vi.fn(),
requeueStaleInProgress: vi.fn(),
markAsInProgress: vi.fn(),
markAsCompleted: vi.fn(),
@@ -322,6 +329,7 @@ describe("MediaService Unit Tests", () => {
const mockMedia = {
id: "new-media-id",
...mockFileInfo,
+ fileSize: mockFileInfo.size,
mediaSourceId: sourceId,
mediaType: "image",
};
diff --git a/apps/server/src/tests/unit/infrastructure/ai/dual-write-ccip-vector-store.test.ts b/apps/server/src/tests/unit/infrastructure/ai/dual-write-ccip-vector-store.test.ts
new file mode 100644
index 000000000..9426b461a
--- /dev/null
+++ b/apps/server/src/tests/unit/infrastructure/ai/dual-write-ccip-vector-store.test.ts
@@ -0,0 +1,130 @@
+import type {
+ CcipEmbeddingKey,
+ CcipVectorCandidate,
+ CcipVectorMetadata,
+ CcipVectorQuery,
+ CcipVectorReadQuery,
+ CcipVectorRecord,
+ ICcipVectorStore,
+} from "@solid-imager/application/ports/ccip-vector-store";
+import { describe, expect, it } from "vite-plus/test";
+import {
+ CcipDualWriteError,
+ DualWriteCcipVectorStore,
+} from "~/infrastructure/ai/dual-write-ccip-vector-store";
+import {
+ CcipStoreReadOnlyError,
+ LanceDbCcipVectorStore,
+} from "~/infrastructure/ai/lancedb-ccip-vector-store";
+
+const record: CcipVectorRecord = {
+ regionId: null,
+ regionKind: "full",
+ mediaId: "11111111-1111-4111-8111-111111111111",
+ mediaSourceId: "22222222-2222-4222-8222-222222222222",
+ vector: [1],
+ model: "model",
+ embeddingVersion: 1,
+ mediaModifiedAt: new Date("2026-07-01T00:00:00.000Z"),
+ inputRevision: "revision",
+ preprocessingProfile: "profile",
+ extractedAt: new Date("2026-07-02T00:00:00.000Z"),
+};
+
+class MemoryCcipStore implements ICcipVectorStore {
+ readonly records = new Map();
+ upsertCalls = 0;
+ failWrites = false;
+
+ async getByRegion(): Promise {
+ return null;
+ }
+
+ async get(mediaId: string): Promise {
+ return this.records.get(mediaId) ?? null;
+ }
+
+ async getMany(mediaIds: string[]): Promise> {
+ return new Map(
+ mediaIds.flatMap((mediaId) => {
+ const value = this.records.get(mediaId);
+ return value ? [[mediaId, value] as const] : [];
+ }),
+ );
+ }
+
+ async getMetadataMany(): Promise> {
+ return new Map();
+ }
+
+ async upsert(value: CcipVectorRecord): Promise {
+ this.upsertCalls += 1;
+ if (this.failWrites) throw new Error("simulated backend failure");
+ this.records.set(value.mediaId, value);
+ }
+
+ async upsertMany(values: CcipVectorRecord[]): Promise {
+ for (const value of values) await this.upsert(value);
+ }
+
+ async delete(mediaId: string): Promise {
+ this.records.delete(mediaId);
+ }
+
+ async deleteRegion(): Promise {}
+ async deleteEmbedding(_key: CcipEmbeddingKey): Promise {}
+ async deleteBySource(): Promise {}
+ async listMediaIds(_query?: CcipVectorQuery): Promise {
+ return [...this.records.keys()];
+ }
+ async list(_query?: CcipVectorQuery): Promise {
+ return [...this.records.values()];
+ }
+ async search(
+ _vector: number[],
+ _limit: number,
+ _query: CcipVectorReadQuery,
+ ): Promise {
+ return [];
+ }
+}
+
+describe("DualWriteCcipVectorStore", () => {
+ it("reports partial success and permits an idempotent retry", async () => {
+ const primary = new MemoryCcipStore();
+ const secondary = new MemoryCcipStore();
+ secondary.failWrites = true;
+ const store = new DualWriteCcipVectorStore(primary, [
+ { name: "primary", store: primary },
+ { name: "secondary", store: secondary },
+ ]);
+
+ const failure = await store.upsert(record).catch((error: unknown) => error);
+ expect(failure).toBeInstanceOf(CcipDualWriteError);
+ expect(failure).toMatchObject({
+ operation: "upsert",
+ succeededBackends: ["primary"],
+ failedBackend: "secondary",
+ });
+ expect(primary.records.size).toBe(1);
+ expect(secondary.records.size).toBe(0);
+
+ secondary.failWrites = false;
+ await store.upsert(record);
+ expect(primary.upsertCalls).toBe(2);
+ expect(primary.records.size).toBe(1);
+ expect(secondary.records.size).toBe(1);
+ });
+});
+
+describe("LanceDbCcipVectorStore read-only mode", () => {
+ it("rejects mutations with a typed error before opening the database", async () => {
+ const store = new LanceDbCcipVectorStore("unused-read-only-test-path", {
+ readOnly: true,
+ });
+
+ await expect(store.upsert(record)).rejects.toBeInstanceOf(
+ CcipStoreReadOnlyError,
+ );
+ });
+});
diff --git a/apps/server/src/tests/unit/infrastructure/ai/remote-crop-request.test.ts b/apps/server/src/tests/unit/infrastructure/ai/remote-crop-request.test.ts
new file mode 100644
index 000000000..d9a69dd0f
--- /dev/null
+++ b/apps/server/src/tests/unit/infrastructure/ai/remote-crop-request.test.ts
@@ -0,0 +1,16 @@
+import { describe, expect, it } from "vite-plus/test";
+import { createRemoteCropRequest } from "~/infrastructure/ai/remote-crop-request";
+
+describe("createRemoteCropRequest", () => {
+ it("forwards the transparent render choice to the remote AI request", () => {
+ const request = createRemoteCropRequest(
+ new Uint8Array([1, 2, 3]),
+ "source.png",
+ true,
+ );
+
+ expect(request.transparent).toBe(true);
+ expect(request.file.name).toBe("source.png");
+ expect(request.file.size).toBe(3);
+ });
+});
diff --git a/apps/server/src/tests/unit/infrastructure/api/jobs-router.test.ts b/apps/server/src/tests/unit/infrastructure/api/jobs-router.test.ts
new file mode 100644
index 000000000..a32d44c68
--- /dev/null
+++ b/apps/server/src/tests/unit/infrastructure/api/jobs-router.test.ts
@@ -0,0 +1,52 @@
+import { safeJobSchema } from "@solid-imager/core/domain/jobs/schemas";
+import type { Job } from "@solid-imager/core/domain/repositories/job-repository";
+import { describe, expect, it } from "vite-plus/test";
+import { toSafeJob } from "~/infrastructure/api/routers/jobs-router";
+
+describe("toSafeJob", () => {
+ it("returns DB-authoritative parent progress without exposing raw job fields", () => {
+ const job: Job = {
+ id: "11111111-1111-4111-8111-111111111111",
+ type: "bulk_tagging_parent",
+ mediaSourceId: "22222222-2222-4222-8222-222222222222",
+ status: "failed",
+ payload: {
+ total: 12,
+ processed: 9,
+ failed: 3,
+ mediaSourceId: "22222222-2222-4222-8222-222222222222",
+ secretPath: "/private/media",
+ },
+ result: { internal: "do not expose" },
+ error: "stack trace with /private/media",
+ createdAt: new Date("2026-07-23T00:00:00.000Z"),
+ updatedAt: new Date("2026-07-23T00:01:00.000Z"),
+ parentId: null,
+ queueName: "default",
+ targetId: null,
+ inputRevision: null,
+ dedupeKey: "private-dedupe-key",
+ concurrencyKey: "private-concurrency-key",
+ availableAt: new Date("2026-07-23T00:00:00.000Z"),
+ attemptCount: 2,
+ maxAttempts: 5,
+ leaseDurationMs: 300_000,
+ claimToken: null,
+ claimedBy: null,
+ claimedAt: null,
+ heartbeatAt: null,
+ errorCode: "JOB_EXECUTION_FAILED",
+ };
+
+ const safeJob = toSafeJob(job);
+
+ expect(safeJobSchema.parse(safeJob)).toEqual(safeJob);
+ expect(safeJob.progress).toEqual({ processed: 9, failed: 3, total: 12 });
+ expect(safeJob.errorMessage).toBe("The job failed.");
+ expect(safeJob).not.toHaveProperty("payload");
+ expect(safeJob).not.toHaveProperty("result");
+ expect(safeJob).not.toHaveProperty("error");
+ expect(safeJob).not.toHaveProperty("dedupeKey");
+ expect(safeJob).not.toHaveProperty("concurrencyKey");
+ });
+});
diff --git a/apps/server/src/tests/unit/infrastructure/api/media-region-api-errors.test.ts b/apps/server/src/tests/unit/infrastructure/api/media-region-api-errors.test.ts
new file mode 100644
index 000000000..b72e0c731
--- /dev/null
+++ b/apps/server/src/tests/unit/infrastructure/api/media-region-api-errors.test.ts
@@ -0,0 +1,21 @@
+import { ORPCError } from "@orpc/server";
+import { StaleMediaRegionError } from "@solid-imager/core/domain/errors";
+import { describe, expect, it } from "vite-plus/test";
+import { toMediaRegionOrpcError } from "~/infrastructure/api/media-region-api-errors";
+
+describe("toMediaRegionOrpcError", () => {
+ it("maps stale materialization conflicts to oRPC CONFLICT (HTTP 409)", () => {
+ try {
+ toMediaRegionOrpcError(
+ new StaleMediaRegionError("10000000-0000-4000-8000-000000000001"),
+ );
+ } catch (error) {
+ expect(error).toBeInstanceOf(ORPCError);
+ if (error instanceof ORPCError) {
+ expect(error.code).toBe("CONFLICT");
+ }
+ return;
+ }
+ throw new Error("Expected a conflict error.");
+ });
+});
diff --git a/apps/server/src/tests/unit/infrastructure/api/media-region-render-handler.test.ts b/apps/server/src/tests/unit/infrastructure/api/media-region-render-handler.test.ts
new file mode 100644
index 000000000..fdf1ea711
--- /dev/null
+++ b/apps/server/src/tests/unit/infrastructure/api/media-region-render-handler.test.ts
@@ -0,0 +1,80 @@
+import { StaleMediaRegionError } from "@solid-imager/core/domain/errors";
+import { describe, expect, it, vi } from "vite-plus/test";
+import {
+ handleMediaRegionRenderRequest,
+ type MediaRegionRenderService,
+} from "~/infrastructure/api/media-region-render-handler";
+
+const REGION_ID = "10000000-0000-4000-8000-000000000001";
+const REVISION = "a".repeat(64);
+const ETAG = `"${"b".repeat(64)}"`;
+
+function createService(): MediaRegionRenderService {
+ return {
+ getRenderIdentity: vi.fn(async () => ({ etag: ETAG })),
+ render: vi.fn(async () => ({
+ bytes: new Uint8Array([1, 2, 3]),
+ format: "webp" as const,
+ width: 1,
+ height: 1,
+ })),
+ };
+}
+
+describe("handleMediaRegionRenderRequest", () => {
+ it("returns 304 without invoking the renderer when the ETag matches", async () => {
+ const service = createService();
+ const response = await handleMediaRegionRenderRequest(
+ new Request(
+ `http://localhost/api/media-regions/${REGION_ID}/render?revision=${REVISION}&transparent=true`,
+ { headers: { "If-None-Match": ETAG } },
+ ),
+ REGION_ID,
+ service,
+ );
+
+ expect(response.status).toBe(304);
+ expect(response.headers.get("Cache-Control")).toBe("private, no-cache");
+ expect(response.headers.get("ETag")).toBe(ETAG);
+ expect(service.getRenderIdentity).toHaveBeenCalledWith(
+ REGION_ID,
+ REVISION,
+ {
+ transparent: true,
+ },
+ );
+ expect(service.render).not.toHaveBeenCalled();
+ });
+
+ it("maps stale source revisions to HTTP 409", async () => {
+ const service = createService();
+ vi.mocked(service.getRenderIdentity).mockRejectedValueOnce(
+ new StaleMediaRegionError(REGION_ID),
+ );
+ const response = await handleMediaRegionRenderRequest(
+ new Request(
+ `http://localhost/api/media-regions/${REGION_ID}/render?revision=${REVISION}`,
+ ),
+ REGION_ID,
+ service,
+ );
+
+ expect(response.status).toBe(409);
+ expect(service.render).not.toHaveBeenCalled();
+ });
+
+ it("rejects untrusted revision strings before creating response headers", async () => {
+ const service = createService();
+ const response = await handleMediaRegionRenderRequest(
+ new Request(
+ `http://localhost/api/media-regions/${REGION_ID}/render?revision=bad%22%0d%0aX-Test%3Ayes`,
+ ),
+ REGION_ID,
+ service,
+ );
+
+ expect(response.status).toBe(400);
+ expect(service.getRenderIdentity).not.toHaveBeenCalled();
+ expect(response.headers.get("X-Test")).toBeNull();
+ });
+});
diff --git a/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts b/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts
index f24ea1b38..479794b78 100644
--- a/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts
+++ b/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts
@@ -16,6 +16,7 @@ const incrementFailedCount = vi.fn();
const jobRepository: IJobRepository = {
create: vi.fn(),
createIfUnique: vi.fn(),
+ createParentWithDispatch: vi.fn(),
findById: vi.fn(),
findPending: vi.fn(),
markAsInProgress: vi.fn(),
@@ -27,13 +28,17 @@ const jobRepository: IJobRepository = {
incrementFailedCount: (...args: Parameters) =>
incrementFailedCount(...args),
claimPending: vi.fn(),
+ heartbeatClaim: vi.fn(),
+ completeClaim: vi.fn(),
+ failClaim: vi.fn(),
+ releaseClaim: vi.fn(),
+ recomputeBatchProgress: vi.fn(),
+ requeueExpiredLeases: vi.fn(),
requeueStaleInProgress: vi.fn(),
};
vi.mock("~/application/registry", () => ({
- services: {
- getJobRepository: () => jobRepository,
- },
+ services: { getJobRepository: () => jobRepository },
}));
vi.mock("~/application/services/ccip-vector-service", () => ({
@@ -57,136 +62,75 @@ vi.mock("~/infrastructure/logger", () => ({
},
}));
-vi.mock("~/infrastructure/db", () => ({
- db: {},
-}));
+vi.mock("~/infrastructure/db", () => ({ db: {} }));
+
+const childJob = {
+ id: "00000000-0000-4000-8000-000000000020",
+ type: "extract_ccip_vector",
+ mediaSourceId: "00000000-0000-4000-8000-000000000001",
+ status: "in_progress" as const,
+ payload: {
+ mediaId: "00000000-0000-4000-8000-000000000030",
+ force: false,
+ },
+ result: null,
+ error: null,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ parentId: "00000000-0000-4000-8000-000000000010",
+};
describe("processCcipExtractionJob", () => {
beforeEach(() => {
vi.clearAllMocks();
- });
-
- it("publishes child completion and completes the parent once", async () => {
extract.mockResolvedValue({
- record: { mediaId: "00000000-0000-4000-8000-000000000030" },
+ record: { mediaId: childJob.payload.mediaId },
skipped: false,
});
- incrementProgress.mockResolvedValue({
- processed: 1,
- failed: 0,
- total: 1,
- });
+ });
- await processCcipExtractionJob({
- id: "00000000-0000-4000-8000-000000000020",
- type: "extract_ccip_vector",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
- payload: {
- mediaId: "00000000-0000-4000-8000-000000000030",
- force: false,
- },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
- parentId: "00000000-0000-4000-8000-000000000010",
- });
+ it("delegates child completion and parent accounting to the worker", async () => {
+ const result = await processCcipExtractionJob(childJob);
- expect(extract).toHaveBeenCalled();
- expect(incrementProgress).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- "00000000-0000-4000-8000-000000000020",
- 1,
- );
- expect(publishJob).toHaveBeenCalledWith("job-completed", {
- jobId: "00000000-0000-4000-8000-000000000020",
- message: "CCIP vector extraction completed",
- });
- expect(publishJob).toHaveBeenCalledWith("job-progress", {
- jobId: "00000000-0000-4000-8000-000000000010",
- processed: 1,
- total: 1,
+ expect(result).toEqual({
+ record: { mediaId: childJob.payload.mediaId },
+ skipped: false,
});
- expect(update).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- {
- status: "completed",
- },
+ expect(extract).toHaveBeenCalledWith(
+ childJob.mediaSourceId,
+ childJob.payload.mediaId,
+ false,
+ undefined,
);
+ expect(incrementProgress).not.toHaveBeenCalled();
+ expect(incrementFailedCount).not.toHaveBeenCalled();
+ expect(update).not.toHaveBeenCalled();
+ expect(publishJob).not.toHaveBeenCalled();
});
- it("skips parent progress when the child was already counted", async () => {
- extract.mockResolvedValue({
- record: { mediaId: "00000000-0000-4000-8000-000000000031" },
- skipped: false,
- });
- incrementProgress.mockResolvedValue(null);
+ it("propagates the worker abort signal into extraction", async () => {
+ const controller = new AbortController();
+ await processCcipExtractionJob(childJob, controller.signal);
- await processCcipExtractionJob({
- id: "00000000-0000-4000-8000-000000000021",
- type: "extract_ccip_vector",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
- payload: {
- mediaId: "00000000-0000-4000-8000-000000000031",
- force: false,
- },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
- parentId: "00000000-0000-4000-8000-000000000011",
- });
-
- expect(jobRepository.findById).not.toHaveBeenCalled();
- expect(update).not.toHaveBeenCalled();
+ expect(extract).toHaveBeenCalledWith(
+ childJob.mediaSourceId,
+ childJob.payload.mediaId,
+ false,
+ controller.signal,
+ );
});
- it("increments failed count and marks parent failed when all children are done", async () => {
+ it("rethrows failures without mutating parent progress from the child handler", async () => {
extract.mockRejectedValue(new Error("ccip error"));
- incrementFailedCount.mockResolvedValue({
- processed: 0,
- failed: 1,
- total: 1,
- });
- await expect(
- processCcipExtractionJob({
- id: "00000000-0000-4000-8000-000000000020",
- type: "extract_ccip_vector",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
- payload: {
- mediaId: "00000000-0000-4000-8000-000000000030",
- force: false,
- },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
- parentId: "00000000-0000-4000-8000-000000000010",
- }),
- ).rejects.toThrow("ccip error");
-
- expect(incrementFailedCount).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- "00000000-0000-4000-8000-000000000020",
- 1,
- );
- expect(update).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- {
- status: "failed",
- },
- );
- expect(publishJob).toHaveBeenCalledWith("job-failed", {
- jobId: "00000000-0000-4000-8000-000000000010",
- error: "1 item(s) failed",
- });
+ await expect(processCcipExtractionJob(childJob)).rejects.toThrow("ccip error");
+ expect(incrementFailedCount).not.toHaveBeenCalled();
+ expect(incrementProgress).not.toHaveBeenCalled();
+ expect(update).not.toHaveBeenCalled();
+ expect(publishJob).not.toHaveBeenCalled();
});
- it("extracts and persists a CCIP batch with item-based parent progress", async () => {
+ it("supports legacy mediaIds payloads without child-owned progress", async () => {
const mediaIds = [
"00000000-0000-4000-8000-000000000031",
"00000000-0000-4000-8000-000000000032",
@@ -194,72 +138,36 @@ describe("processCcipExtractionJob", () => {
extractBatch.mockResolvedValue(
mediaIds.map((mediaId) => ({
status: "fulfilled",
- value: {
- mediaId,
- record: { mediaId },
- skipped: false,
- },
+ value: { mediaId, record: { mediaId }, skipped: false },
})),
);
- incrementProgress.mockResolvedValue({
- processed: 2,
- failed: 0,
- total: 2,
- });
await processCcipExtractionJob({
+ ...childJob,
id: "00000000-0000-4000-8000-000000000022",
- type: "extract_ccip_vector",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
payload: { mediaIds, force: false },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
- parentId: "00000000-0000-4000-8000-000000000012",
});
expect(extractBatch).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000001",
+ childJob.mediaSourceId,
mediaIds,
false,
1,
+ undefined,
);
- expect(incrementProgress).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000012",
- "00000000-0000-4000-8000-000000000022",
- 2,
- );
- expect(update).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000012",
- {
- status: "completed",
- },
- );
+ expect(incrementProgress).not.toHaveBeenCalled();
+ expect(update).not.toHaveBeenCalled();
+ expect(publishJob).not.toHaveBeenCalled();
});
});
describe("processBatchCcipDispatchJob", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- });
-
it("throws when parentId is missing", async () => {
await expect(
processBatchCcipDispatchJob({
- id: "00000000-0000-4000-8000-000000000100",
+ ...childJob,
type: "batch_ccip_dispatch",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
- payload: {
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- force: false,
- },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
+ payload: { mediaSourceId: childJob.mediaSourceId, force: false },
parentId: null,
}),
).rejects.toThrow("batch_ccip_dispatch requires parentId");
diff --git a/apps/server/src/tests/unit/infrastructure/jobs/job-worker.test.ts b/apps/server/src/tests/unit/infrastructure/jobs/job-worker.test.ts
index 4726667a5..e206233d1 100644
--- a/apps/server/src/tests/unit/infrastructure/jobs/job-worker.test.ts
+++ b/apps/server/src/tests/unit/infrastructure/jobs/job-worker.test.ts
@@ -9,6 +9,7 @@ import {
} from "vite-plus/test";
import type { IJobRepository } from "~/domain/repositories/job-repository";
import type { Job } from "~/infrastructure/db/schema";
+import { NonRetryableJobError } from "~/infrastructure/jobs/job-errors";
import { JobWorker } from "~/infrastructure/jobs/job-worker";
// Mock logger to avoid noise
@@ -26,11 +27,38 @@ vi.mock("~/infrastructure/logger", () => ({
describe("JobWorker", () => {
let jobRepo: IJobRepository;
- let processor: (job: Job) => Promise;
+ let processor: (job: Job, signal?: AbortSignal) => Promise;
let worker: JobWorker;
const TimerDelay = 100;
const TotalExpectedCalls = 3; // AI + 2 Normal
+ const makeClaimedJob = (overrides: Partial): Job => ({
+ id: "11111111-1111-4111-8111-111111111111",
+ type: "processMedia",
+ mediaSourceId: "22222222-2222-4222-8222-222222222222",
+ status: "in_progress",
+ payload: {},
+ result: null,
+ error: null,
+ createdAt: new Date("2026-01-01T00:00:00.000Z"),
+ updatedAt: new Date("2026-01-01T00:00:00.000Z"),
+ parentId: null,
+ queueName: "default",
+ targetId: null,
+ inputRevision: null,
+ dedupeKey: null,
+ concurrencyKey: null,
+ availableAt: new Date("2026-01-01T00:00:00.000Z"),
+ attemptCount: 1,
+ maxAttempts: 5,
+ leaseDurationMs: 300_000,
+ claimToken: "33333333-3333-4333-8333-333333333333",
+ claimedBy: "test-worker",
+ claimedAt: new Date("2026-01-01T00:00:00.000Z"),
+ heartbeatAt: new Date("2026-01-01T00:00:00.000Z"),
+ errorCode: null,
+ ...overrides,
+ });
beforeEach(() => {
vi.useFakeTimers();
@@ -39,9 +67,18 @@ describe("JobWorker", () => {
jobRepo = {
create: vi.fn(),
createIfUnique: vi.fn(),
+ createParentWithDispatch: vi.fn(),
findById: vi.fn(),
findPending: vi.fn().mockResolvedValue([]),
claimPending: vi.fn().mockResolvedValue([]),
+ heartbeatClaim: vi.fn().mockResolvedValue(true),
+ completeClaim: vi.fn().mockResolvedValue(true),
+ failClaim: vi
+ .fn()
+ .mockResolvedValue({ status: "failed", attemptCount: 1 }),
+ releaseClaim: vi.fn().mockResolvedValue(true),
+ recomputeBatchProgress: vi.fn().mockResolvedValue(null),
+ requeueExpiredLeases: vi.fn().mockResolvedValue(0),
requeueStaleInProgress: vi.fn().mockResolvedValue(0),
markAsInProgress: vi.fn().mockResolvedValue(undefined),
markAsCompleted: vi.fn().mockResolvedValue(undefined),
@@ -68,14 +105,12 @@ describe("JobWorker", () => {
} as AppConfig);
// Mock 5 pending normal jobs
- const normalJobs = Array.from(
- { length: 5 },
- (_, i) =>
- ({
+ const normalJobs = Array.from({ length: 5 }, (_, i) =>
+ makeClaimedJob({
id: `job-${i}`,
type: "normal_job",
status: "pending",
- }) as Job,
+ }),
);
// Mock claimPending to return jobs
@@ -109,14 +144,13 @@ describe("JobWorker", () => {
} as AppConfig);
// Mock 3 pending AI jobs
- const aiJobs = Array.from(
- { length: 3 },
- (_, i) =>
- ({
+ const aiJobs = Array.from({ length: 3 }, (_, i) =>
+ makeClaimedJob({
id: `ai-job-${i}`,
type: "auto_tagging",
status: "pending",
- }) as Job,
+ queueName: "ai",
+ }),
);
// Mock claimPending
@@ -142,6 +176,7 @@ describe("JobWorker", () => {
expect(processor).toHaveBeenCalledTimes(1);
expect(processor).toHaveBeenCalledWith(
expect.objectContaining({ id: "ai-job-0" }),
+ expect.any(AbortSignal),
);
});
@@ -151,21 +186,22 @@ describe("JobWorker", () => {
jobs: { concurrency: 2, aiConcurrency: 1, pollIntervalMs: 1000 },
} as AppConfig);
- const aiJob = {
+ const aiJob = makeClaimedJob({
id: "ai-1",
type: "auto_tagging",
status: "pending",
- } as Job;
- const normalJob1 = {
+ queueName: "ai",
+ });
+ const normalJob1 = makeClaimedJob({
id: "normal-1",
type: "normal",
status: "pending",
- } as Job;
- const normalJob2 = {
+ });
+ const normalJob2 = makeClaimedJob({
id: "normal-2",
type: "normal",
status: "pending",
- } as Job;
+ });
// Mock claimPending
(jobRepo.claimPending as any).mockImplementation(
@@ -207,11 +243,11 @@ describe("JobWorker", () => {
jobs: { concurrency: 1, aiConcurrency: 1, pollIntervalMs: 1000 },
} as AppConfig);
- const customJob = {
+ const customJob = makeClaimedJob({
id: "custom-1",
type: "custom",
status: "pending",
- } as Job;
+ });
processor = vi
.fn()
@@ -233,11 +269,19 @@ describe("JobWorker", () => {
worker.start();
await vi.advanceTimersByTimeAsync(TimerDelay);
- expect(processor).toHaveBeenCalledWith(customJob);
- expect(jobRepo.markAsCompleted).toHaveBeenCalledWith("custom-1", {
- success: true,
- parentProcessed: true,
- });
+ expect(processor).toHaveBeenCalledWith(
+ customJob,
+ expect.any(AbortSignal),
+ );
+ expect(jobRepo.completeClaim).toHaveBeenCalledWith(
+ "custom-1",
+ {
+ claimToken: customJob.claimToken,
+ inputRevision: customJob.inputRevision,
+ },
+ { success: true, parentProcessed: true },
+ );
+ expect(jobRepo.markAsCompleted).not.toHaveBeenCalled();
});
it("should requeue overlapping claimed LanceDB sync jobs per media source", async () => {
@@ -257,24 +301,24 @@ describe("JobWorker", () => {
jobs: { concurrency: 3, aiConcurrency: 1, pollIntervalMs: 1000 },
} as AppConfig);
- const fullSyncJob = {
+ const fullSyncJob = makeClaimedJob({
id: "lancedb-full-1",
type: "sync_lancedb_full",
mediaSourceId: "source-1",
status: "pending",
- } as Job;
- const deltaSyncSameSourceJob = {
+ });
+ const deltaSyncSameSourceJob = makeClaimedJob({
id: "lancedb-delta-1",
type: "sync_lancedb_delta",
mediaSourceId: "source-1",
status: "pending",
- } as Job;
- const deltaSyncOtherSourceJob = {
+ });
+ const deltaSyncOtherSourceJob = makeClaimedJob({
id: "lancedb-delta-2",
type: "sync_lancedb_delta",
mediaSourceId: "source-2",
status: "pending",
- } as Job;
+ });
(jobRepo.claimPending as any).mockImplementation(
(limit: number, options: any) => {
@@ -295,12 +339,26 @@ describe("JobWorker", () => {
await vi.advanceTimersByTimeAsync(TimerDelay);
expect(processor).toHaveBeenCalledTimes(2);
- expect(processor).toHaveBeenCalledWith(fullSyncJob);
- expect(processor).not.toHaveBeenCalledWith(deltaSyncSameSourceJob);
- expect(processor).toHaveBeenCalledWith(deltaSyncOtherSourceJob);
- expect(jobRepo.update).toHaveBeenCalledWith(deltaSyncSameSourceJob.id, {
- status: "pending",
- });
+ expect(processor).toHaveBeenCalledWith(
+ fullSyncJob,
+ expect.any(AbortSignal),
+ );
+ expect(processor).not.toHaveBeenCalledWith(
+ deltaSyncSameSourceJob,
+ expect.any(AbortSignal),
+ );
+ expect(processor).toHaveBeenCalledWith(
+ deltaSyncOtherSourceJob,
+ expect.any(AbortSignal),
+ );
+ expect(jobRepo.releaseClaim).toHaveBeenCalledWith(
+ deltaSyncSameSourceJob.id,
+ {
+ claimToken: deltaSyncSameSourceJob.claimToken,
+ inputRevision: deltaSyncSameSourceJob.inputRevision,
+ },
+ );
+ expect(jobRepo.update).not.toHaveBeenCalled();
resolveProcessor();
await vi.runOnlyPendingTimersAsync();
@@ -311,12 +369,12 @@ describe("JobWorker", () => {
jobs: { concurrency: 3, aiConcurrency: 1, pollIntervalMs: 1000 },
} as AppConfig);
- const syncJob = {
+ const syncJob = makeClaimedJob({
id: "lancedb-sync-1",
type: "sync_lancedb",
mediaSourceId: "source-active",
status: "pending",
- } as Job;
+ });
let resolveProcessor: () => void = () => {};
processor = vi.fn(
@@ -348,7 +406,10 @@ describe("JobWorker", () => {
await vi.advanceTimersByTimeAsync(TimerDelay);
expect(processor).toHaveBeenCalledTimes(1);
- expect(processor).toHaveBeenCalledWith(syncJob);
+ expect(processor).toHaveBeenCalledWith(
+ syncJob,
+ expect.any(AbortSignal),
+ );
// Advance to trigger second poll while syncJob is still active
await vi.advanceTimersByTimeAsync(1000);
@@ -363,4 +424,139 @@ describe("JobWorker", () => {
resolveProcessor();
await vi.runOnlyPendingTimersAsync();
});
+
+ it("aborts processing and discards output when the heartbeat loses its lease", async () => {
+ const claimedJob = makeClaimedJob({ id: "lease-lost-1" });
+ let returned = false;
+ (jobRepo.claimPending as any).mockImplementation(
+ (_limit: number, options: { excludeTypes?: string[] }) => {
+ if (options.excludeTypes && !returned) {
+ returned = true;
+ return Promise.resolve([claimedJob]);
+ }
+ return Promise.resolve([]);
+ },
+ );
+ (jobRepo.heartbeatClaim as any).mockResolvedValue(false);
+ processor = vi.fn(
+ (_job: Job, signal?: AbortSignal) =>
+ new Promise((resolve) => {
+ signal?.addEventListener("abort", () => resolve({ ignored: true }), {
+ once: true,
+ });
+ }),
+ );
+ worker = new JobWorker(jobRepo, processor);
+
+ worker.start();
+ await vi.advanceTimersByTimeAsync(TimerDelay);
+ await vi.advanceTimersByTimeAsync(30_000);
+
+ expect(jobRepo.heartbeatClaim).toHaveBeenCalledWith("lease-lost-1", {
+ claimToken: claimedJob.claimToken,
+ inputRevision: claimedJob.inputRevision,
+ });
+ expect(jobRepo.completeClaim).not.toHaveBeenCalled();
+ expect(jobRepo.failClaim).not.toHaveBeenCalled();
+ });
+
+ it("schedules retryable failures through the fenced repository transition", async () => {
+ const claimedJob = makeClaimedJob({ id: "retryable-1", attemptCount: 2 });
+ let returned = false;
+ (jobRepo.claimPending as any).mockImplementation(
+ (_limit: number, options: { excludeTypes?: string[] }) => {
+ if (options.excludeTypes && !returned) {
+ returned = true;
+ return Promise.resolve([claimedJob]);
+ }
+ return Promise.resolve([]);
+ },
+ );
+ (jobRepo.failClaim as any).mockResolvedValue({
+ status: "pending",
+ attemptCount: 2,
+ });
+ processor = vi.fn().mockRejectedValue(new Error("temporary"));
+ worker = new JobWorker(jobRepo, processor);
+
+ worker.start();
+ await vi.advanceTimersByTimeAsync(TimerDelay);
+
+ expect(jobRepo.failClaim).toHaveBeenCalledWith(
+ "retryable-1",
+ {
+ claimToken: claimedJob.claimToken,
+ inputRevision: claimedJob.inputRevision,
+ },
+ expect.objectContaining({
+ error: "temporary",
+ errorCode: "JOB_EXECUTION_FAILED",
+ retryable: true,
+ retryAt: expect.any(Date),
+ }),
+ );
+ expect(jobRepo.recomputeBatchProgress).not.toHaveBeenCalled();
+ });
+
+ it("marks validation failures as non-retryable", async () => {
+ const claimedJob = makeClaimedJob({ id: "invalid-1" });
+ let returned = false;
+ (jobRepo.claimPending as any).mockImplementation(
+ (_limit: number, options: { excludeTypes?: string[] }) => {
+ if (options.excludeTypes && !returned) {
+ returned = true;
+ return Promise.resolve([claimedJob]);
+ }
+ return Promise.resolve([]);
+ },
+ );
+ processor = vi
+ .fn()
+ .mockRejectedValue(
+ new NonRetryableJobError("INVALID_JOB_PAYLOAD", "invalid payload"),
+ );
+ worker = new JobWorker(jobRepo, processor);
+
+ worker.start();
+ await vi.advanceTimersByTimeAsync(TimerDelay);
+
+ expect(jobRepo.failClaim).toHaveBeenCalledWith(
+ "invalid-1",
+ expect.any(Object),
+ expect.objectContaining({
+ errorCode: "INVALID_JOB_PAYLOAD",
+ retryable: false,
+ }),
+ );
+ });
+
+ it("recomputes parent progress only after a child completes terminally", async () => {
+ const claimedJob = makeClaimedJob({
+ id: "child-1",
+ parentId: "parent-1",
+ });
+ let returned = false;
+ (jobRepo.claimPending as any).mockImplementation(
+ (_limit: number, options: { excludeTypes?: string[] }) => {
+ if (options.excludeTypes && !returned) {
+ returned = true;
+ return Promise.resolve([claimedJob]);
+ }
+ return Promise.resolve([]);
+ },
+ );
+ (jobRepo.recomputeBatchProgress as any).mockResolvedValue({
+ processed: 1,
+ failed: 0,
+ total: 1,
+ status: "completed",
+ transitioned: true,
+ });
+
+ worker.start();
+ await vi.advanceTimersByTimeAsync(TimerDelay);
+
+ expect(jobRepo.completeClaim).toHaveBeenCalled();
+ expect(jobRepo.recomputeBatchProgress).toHaveBeenCalledWith("parent-1");
+ });
});
diff --git a/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts b/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts
index 2314b2c7f..1aab616d7 100644
--- a/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts
+++ b/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts
@@ -17,6 +17,7 @@ const jobRepository: IJobRepository = {
create: vi.fn(),
createIfUnique: (...args: Parameters) =>
createIfUnique(...args),
+ createParentWithDispatch: vi.fn(),
findById: (...args: Parameters) => findById(...args),
findPending: vi.fn(),
markAsInProgress: vi.fn(),
@@ -28,6 +29,12 @@ const jobRepository: IJobRepository = {
incrementFailedCount: (...args: Parameters) =>
incrementFailedCount(...args),
claimPending: vi.fn(),
+ heartbeatClaim: vi.fn(),
+ completeClaim: vi.fn(),
+ failClaim: vi.fn(),
+ releaseClaim: vi.fn(),
+ recomputeBatchProgress: vi.fn(),
+ requeueExpiredLeases: vi.fn(),
requeueStaleInProgress: vi.fn(),
};
@@ -44,9 +51,7 @@ vi.mock("~/application/services/tagging-service", () => ({
},
}));
-vi.mock("~/infrastructure/db", () => ({
- db: {},
-}));
+vi.mock("~/infrastructure/db", () => ({ db: {} }));
vi.mock("~/infrastructure/events/realtime-event-bus", () => ({
RealtimeEventBus: {
@@ -55,13 +60,25 @@ vi.mock("~/infrastructure/events/realtime-event-bus", () => ({
}));
vi.mock("~/infrastructure/logger", () => ({
- logger: {
- error: vi.fn(),
- info: vi.fn(),
- warn: vi.fn(),
- },
+ logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn() },
}));
+const childJob = {
+ id: "00000000-0000-4000-8000-000000000020",
+ type: "auto_tagging",
+ mediaSourceId: "00000000-0000-4000-8000-000000000001",
+ status: "in_progress" as const,
+ payload: {
+ mediaId: "00000000-0000-4000-8000-000000000030",
+ force: false,
+ },
+ result: null,
+ error: null,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ parentId: "00000000-0000-4000-8000-000000000010",
+};
+
describe("processAutoTaggingJob", () => {
beforeEach(() => {
vi.clearAllMocks();
@@ -72,142 +89,63 @@ describe("processAutoTaggingJob", () => {
ips_mapping: {},
});
createIfUnique.mockResolvedValue(null);
- incrementProgress.mockResolvedValue(null);
- incrementFailedCount.mockResolvedValue(null);
});
- it("does not re-publish parent progress when the child was already counted", async () => {
- await processAutoTaggingJob({
- id: "00000000-0000-4000-8000-000000000020",
- type: "auto_tagging",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
+ it("delegates parent accounting to the worker after the child is terminal", async () => {
+ await processAutoTaggingJob(childJob);
+
+ expect(getTagsForMedia).toHaveBeenCalledWith(
+ childJob.mediaSourceId,
+ childJob.payload.mediaId,
+ { signal: undefined, skipCache: false },
+ );
+ expect(createIfUnique).toHaveBeenCalledWith({
+ type: "sync_lancedb_delta",
+ mediaSourceId: childJob.mediaSourceId,
payload: {
- mediaId: "00000000-0000-4000-8000-000000000030",
- force: false,
+ reason: "auto_tagging",
+ mediaIds: [childJob.payload.mediaId],
},
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
- parentId: "00000000-0000-4000-8000-000000000010",
});
-
- expect(incrementProgress).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- "00000000-0000-4000-8000-000000000020",
- );
+ expect(incrementProgress).not.toHaveBeenCalled();
+ expect(incrementFailedCount).not.toHaveBeenCalled();
expect(findById).not.toHaveBeenCalled();
+ expect(update).not.toHaveBeenCalled();
expect(publishJob).not.toHaveBeenCalled();
});
- it("publishes parent progress and completes the parent once", async () => {
- incrementProgress.mockResolvedValue({
- processed: 1,
- failed: 0,
- total: 1,
- });
-
- await processAutoTaggingJob({
- id: "00000000-0000-4000-8000-000000000020",
- type: "auto_tagging",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
- payload: {
- mediaId: "00000000-0000-4000-8000-000000000030",
- force: false,
- },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
- parentId: "00000000-0000-4000-8000-000000000010",
- });
-
- expect(incrementProgress).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- "00000000-0000-4000-8000-000000000020",
+ it("propagates the worker abort signal into the AI operation", async () => {
+ const controller = new AbortController();
+ await processAutoTaggingJob(
+ { ...childJob, payload: { ...childJob.payload, force: true } },
+ controller.signal,
);
- expect(publishJob).toHaveBeenCalledWith("job-progress", {
- jobId: "00000000-0000-4000-8000-000000000010",
- processed: 1,
- total: 1,
- });
- expect(update).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- {
- status: "completed",
- },
+
+ expect(getTagsForMedia).toHaveBeenCalledWith(
+ childJob.mediaSourceId,
+ childJob.payload.mediaId,
+ { signal: controller.signal, skipCache: true },
);
- expect(publishJob).toHaveBeenCalledWith("job-completed", {
- jobId: "00000000-0000-4000-8000-000000000010",
- message: "Batch tagging completed",
- });
});
- it("increments failed count and marks parent failed when all children are done", async () => {
+ it("rethrows failures without mutating parent progress from the child handler", async () => {
getTagsForMedia.mockRejectedValue(new Error("tagging error"));
- incrementFailedCount.mockResolvedValue({
- processed: 0,
- failed: 1,
- total: 1,
- });
- await expect(
- processAutoTaggingJob({
- id: "00000000-0000-4000-8000-000000000020",
- type: "auto_tagging",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
- payload: {
- mediaId: "00000000-0000-4000-8000-000000000030",
- force: false,
- },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
- parentId: "00000000-0000-4000-8000-000000000010",
- }),
- ).rejects.toThrow("tagging error");
-
- expect(incrementFailedCount).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- "00000000-0000-4000-8000-000000000020",
- );
- expect(update).toHaveBeenCalledWith(
- "00000000-0000-4000-8000-000000000010",
- {
- status: "failed",
- },
- );
- expect(publishJob).toHaveBeenCalledWith("job-failed", {
- jobId: "00000000-0000-4000-8000-000000000010",
- error: "1 child job(s) failed",
- });
+ await expect(processAutoTaggingJob(childJob)).rejects.toThrow("tagging error");
+ expect(incrementFailedCount).not.toHaveBeenCalled();
+ expect(incrementProgress).not.toHaveBeenCalled();
+ expect(update).not.toHaveBeenCalled();
+ expect(publishJob).not.toHaveBeenCalled();
});
});
describe("processBulkTaggingDispatchJob", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- });
-
it("throws when parentId is missing", async () => {
await expect(
processBulkTaggingDispatchJob({
- id: "00000000-0000-4000-8000-000000000100",
+ ...childJob,
type: "bulk_tagging_dispatch",
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- status: "in_progress",
- payload: {
- mediaSourceId: "00000000-0000-4000-8000-000000000001",
- force: false,
- },
- result: null,
- error: null,
- createdAt: new Date(),
- updatedAt: new Date(),
+ payload: { mediaSourceId: childJob.mediaSourceId, force: false },
parentId: null,
}),
).rejects.toThrow("bulk_tagging_dispatch requires parentId");
diff --git a/apps/server/src/tests/unit/media/copy-media-job.test.ts b/apps/server/src/tests/unit/media/copy-media-job.test.ts
index 9ef0bfb4c..677ea5449 100644
--- a/apps/server/src/tests/unit/media/copy-media-job.test.ts
+++ b/apps/server/src/tests/unit/media/copy-media-job.test.ts
@@ -229,6 +229,7 @@ describe("Reproduction: Copy Media Job Type", () => {
width: 800,
height: 600,
fileSize: 1024,
+ modifiedAt: new Date("2026-01-01T00:00:00.000Z"),
};
// Mock MediaRepository.findById (used by MediaService.copyMedia)
diff --git a/apps/server/vite.config.ts b/apps/server/vite.config.ts
index 8388f9d44..446f02095 100644
--- a/apps/server/vite.config.ts
+++ b/apps/server/vite.config.ts
@@ -295,7 +295,6 @@ export default defineConfig({
"dghs-imgutils-rs",
"ffmpeg-static",
"fluent-ffmpeg",
- "@electric-sql/pglite",
"archiver",
"@lancedb/lancedb",
"apache-arrow",
@@ -310,12 +309,13 @@ export default defineConfig({
"@tanstack/solid-start",
"@kobalte/core",
"solid-sonner",
+ "@electric-sql/pglite",
+ "@electric-sql/pglite-pgvector",
"corvu",
"@solid-primitives/.*",
],
external: [
"bun",
- "@electric-sql/pglite",
"ffmpeg-static",
"ffmpeg-static-static",
"fluent-ffmpeg",
diff --git a/apps/tauri/src/components/media/character-crop-modal.tsx b/apps/tauri/src/components/media/character-crop-modal.tsx
index 7fc67abe5..8c0f92b75 100644
--- a/apps/tauri/src/components/media/character-crop-modal.tsx
+++ b/apps/tauri/src/components/media/character-crop-modal.tsx
@@ -1,6 +1,18 @@
import type { MediaDetails } from "@solid-imager/core/domain/media/schemas";
+import type { SafeMediaRegion } from "@solid-imager/core/domain/media-regions/schemas";
import { CharacterCropModal as SharedCharacterCropModal } from "@solid-imager/ui/character-crop-modal";
-import { serverOrpc } from "~/infrastructure/api-clients/server-orpc-client";
+import {
+ serverApiBaseUrl,
+ serverOrpc,
+} from "~/infrastructure/api-clients/server-orpc-client";
+
+function getRenderUrl(region: SafeMediaRegion, transparent: boolean): string {
+ const query = new URLSearchParams({
+ revision: region.regionRevision,
+ transparent: String(transparent),
+ });
+ return `${serverApiBaseUrl}/api/media-regions/${encodeURIComponent(region.id)}/render?${query}`;
+}
type CharacterCropModalProps = {
isOpen: boolean;
@@ -11,15 +23,35 @@ type CharacterCropModalProps = {
export function CharacterCropModal(props: CharacterCropModalProps) {
return (
{
- return serverOrpc.ai.detectAndCropCharacters({
+ createManualRegion={(input) =>
+ serverOrpc.mediaRegions.createManual(input)
+ }
+ deleteRegion={async (regionId, expectedRevision) => {
+ await serverOrpc.mediaRegions.delete({ regionId, expectedRevision });
+ }}
+ detectRegions={async (mediaId: string) => {
+ const result = await serverOrpc.ai.detectAndCropCharacters({
mediaId,
- transparent,
+ transparent: false,
});
+ if (result.mode !== "media-backed") {
+ throw new Error("Character detection did not return saved regions.");
+ }
+ return result.regions;
}}
+ getRenderUrl={getRenderUrl}
isOpen={props.isOpen}
+ loadRegions={(mediaId) => serverOrpc.mediaRegions.list({ mediaId })}
+ materializeRegion={(regionId, expectedRevision, transparent) =>
+ serverOrpc.mediaRegions.materialize({
+ regionId,
+ expectedRevision,
+ profile: { transparent },
+ })
+ }
media={props.media}
onClose={props.onClose}
+ updateRegion={(input) => serverOrpc.mediaRegions.update(input)}
/>
);
}
diff --git a/apps/tauri/src/infrastructure/api-clients/server-orpc-client.ts b/apps/tauri/src/infrastructure/api-clients/server-orpc-client.ts
index 813d97822..527d07c2b 100644
--- a/apps/tauri/src/infrastructure/api-clients/server-orpc-client.ts
+++ b/apps/tauri/src/infrastructure/api-clients/server-orpc-client.ts
@@ -7,6 +7,8 @@ const SERVER_URL = isDev
? window.location.origin
: import.meta.env.VITE_API_URL || "http://192.168.1.150:3000";
+export const serverApiBaseUrl = SERVER_URL;
+
const tauriFetchAdapter = (
request: Request,
init?: RequestInit & { redirect?: Request["redirect"] },
diff --git a/compose.pg18-rehearsal.yml b/compose.pg18-rehearsal.yml
new file mode 100644
index 000000000..b6d83909a
--- /dev/null
+++ b/compose.pg18-rehearsal.yml
@@ -0,0 +1,23 @@
+services:
+ db-pg18-rehearsal:
+ image: pgvector/pgvector:0.8.5-pg18-bookworm@sha256:12a379b47ad65289572ea0756efc11b7c241a6662833e8af7038cd3b73d647e0
+ restart: "no"
+ environment:
+ POSTGRES_USER: ${DB_USER:-postgres}
+ POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres}
+ POSTGRES_DB: ${DB_DATABASE:-solid-imager}
+ volumes:
+ - pg18-rehearsal-data:/var/lib/postgresql
+ ports:
+ - ${PG18_REHEARSAL_PORT:-55432}:5432
+ healthcheck:
+ test:
+ - CMD-SHELL
+ - pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
+ interval: 2s
+ timeout: 3s
+ retries: 30
+
+volumes:
+ pg18-rehearsal-data:
+ name: solid-imager-pg18-rehearsal-data
diff --git a/docs/runbooks/database-modernization-cutover.md b/docs/runbooks/database-modernization-cutover.md
new file mode 100644
index 000000000..772a26ff6
--- /dev/null
+++ b/docs/runbooks/database-modernization-cutover.md
@@ -0,0 +1,664 @@
+# データストア刷新・移行 runbook
+
+この文書は Issue #613（PostgreSQL 18）、#616（CCIP の LanceDB から
+pgvector への移行）、#619（background jobs の耐障害化）を、本番データに
+適用するときの統合手順である。PostgreSQL 18 の単体リハーサルで使うコマンドと
+各スクリプトの安全策は
+[`postgresql-18-rehearsal.md`](./postgresql-18-rehearsal.md) も参照する。
+
+PostgreSQL major cutover の gate と CCIP read cutover の gate は独立している。
+同じ maintenance window で同時に切り替えず、それぞれの go/no-go と rollback
+boundary を個別に承認する。CCIP の 7 日 observation は CCIP read switch 後の条件で
+あり、PG18 cutover の前提条件ではない。
+
+> **現在の状態:** コードとリハーサル手順を用意した段階であり、実データの
+> cutover は実行していない。本番 DB、LanceDB、volume、compose 定義を変更する
+> 操作は、この文書とは別の変更申請・承認・担当者立会いを必要とする。
+
+## 変更しない境界
+
+- job の永続化先は単一の `jobs` テーブルである。job type ごとの中間テーブルは
+ 作らない。型、queue、dedupe、concurrency、retry、lease の差は job registry と
+ `jobs` の列で表現する。
+- `compose.pg18-rehearsal.yml` は隔離リハーサル専用である。既定の
+ `compose.yml`、`db-data/`、本番 volume を参照させない。
+- legacy LanceDB の source dump/snapshot は読み取り専用の証拠物であり、移行先、
+ rollback mirror、checkpoint の置き場として再利用しない。
+- PostgreSQL 17 の data directory を PostgreSQL 18 から直接開かない。major
+ upgrade は custom-format dump/restore だけで行う。
+- 既存ファイル、dump、report、checkpoint を上書きしない。再試行では新しい
+ run ID と空の移行先を使う。
+
+## 1. 実行記録と前提条件
+
+一回のリハーサルまたは cutover ごとに、UTC の run ID と権限を限定した記録
+ディレクトリを作る。以下の変数名は例であり、秘密情報は記録ファイルや shell
+history に書かない。
+
+```bash
+RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
+RUN_DIR="/var/tmp/solid-imager-cutover-${RUN_ID}"
+install -d -m 0700 "${RUN_DIR}"
+date --iso-8601=seconds
+git rev-parse HEAD
+```
+
+次を作業記録へ残す。
+
+| 項目 | 必須記録 |
+| --- | --- |
+| 識別 | run ID、Issue/変更申請、commit SHA、担当者、承認者 |
+| 時刻 | 各 phase の開始・終了、write-freeze、最初の PG18 write、read switch |
+| 容量 | PG17 data、PG18 volume、Lance source/snapshot/rollback、dump の bytes |
+| 所要時間 | dump、restore、migration、`ANALYZE`/validation、CCIP backfill/parity |
+| image | compose に記載した tag、稼働 container の image ID/digest、RepoDigest |
+| 検証 | source/target JSON report、manifest、checkpoint、parity report、ログ |
+| 判断 | 各 go/no-go の判定者、時刻、根拠、未解決事項 |
+
+必要なローカルコマンドは `docker`（Compose v2）、`bun`、`git`、`jq`、
+`sha256sum`、`/usr/bin/time` である。開始前に application server と worker を
+個別に停止・起動できること、DB と LanceDB の所有者、監視方法、連絡先を確認する。
+
+PostgreSQL image は tag だけで判定しない。リハーサルで、実際に稼働した image
+の組を保存する。
+
+```bash
+docker compose -f compose.yml config --images
+docker inspect --format '{{.Config.Image}}|{{.Image}}' \
+ "$(docker compose -f compose.yml ps -q db)"
+docker image inspect pgvector/pgvector:pg17 \
+ --format '{{json .RepoDigests}}'
+
+docker compose -f compose.pg18-rehearsal.yml config --images
+docker image inspect \
+ 'pgvector/pgvector:0.8.5-pg18-bookworm@sha256:12a379b47ad65289572ea0756efc11b7c241a6662833e8af7038cd3b73d647e0' \
+ --format '{{json .RepoDigests}}'
+```
+
+本番 PG18 compose は、リハーサルと同じ
+`tag@sha256:12a379b...d647e0` を使い、稼働 container の tag と image ID/digest
+がリハーサル記録と完全一致しなければならない。一致しない場合は no-go として
+新しい image でリハーサルをやり直す。
+
+## 2. PostgreSQL 17 → 18 timed rehearsal
+
+### 2.1 事前条件
+
+1. PG17 source の現在の migration level を記録する。PG18 用 schema migration を
+ rehearsal のためだけに PG17 へ先行適用しない。
+2. PG18 restore 後に final code revision の `db:migrate` を明示的に実行する。
+ source migration 列は target の完全な prefix でなければならない。
+3. `compose.pg18-rehearsal.yml` の target image と volume mount
+ (`/var/lib/postgresql`) を review する。
+4. source と target の空き容量が、source data と custom dump を保持しても十分で
+ あることを確認する。
+
+### 2.2 source baseline と custom dump
+
+PG17 source の構造・件数を JSON に固定する。現行 source は vector extension を
+持たない前提なので、その不在も baseline として検証する。
+
+```bash
+/usr/bin/time -v bun apps/server/scripts/validate-postgres-rehearsal.ts \
+ --compose-file compose.yml \
+ --service db \
+ --expected-major 17 \
+ --expect-vector-unavailable \
+ --output "${RUN_DIR}/pg17-source.json"
+
+/usr/bin/time -v bun apps/server/scripts/dump-db.ts \
+ --compose-file compose.yml \
+ --service db \
+ --output "${RUN_DIR}/pg17-source.dump"
+
+stat --format '%n %s bytes' "${RUN_DIR}/pg17-source.dump"
+sha256sum "${RUN_DIR}/pg17-source.dump" \
+ > "${RUN_DIR}/pg17-source.dump.sha256"
+du -sb db-data
+```
+
+`dump-db.ts` は custom format、TTY 無効、partial file からの atomic rename、
+既存出力の上書き拒否を行う。終了 code、所要時間、bytes、SHA-256 を保存する。
+
+### 2.3 空の PG18 volume へ restore
+
+同名の rehearsal volume が残っている場合は使い回さない。削除は中身と対象名を
+確認し、rehearsal の不要が承認された場合だけ行う。`down -v` は使用しない。
+
+```bash
+docker compose -f compose.pg18-rehearsal.yml up -d --wait db-pg18-rehearsal
+docker compose -f compose.pg18-rehearsal.yml ps db-pg18-rehearsal
+docker inspect --format '{{.Config.Image}}|{{.Image}}' \
+ "$(docker compose -f compose.pg18-rehearsal.yml ps -q db-pg18-rehearsal)"
+docker inspect --format '{{range .Mounts}}{{println .Name .Destination}}{{end}}' \
+ "$(docker compose -f compose.pg18-rehearsal.yml ps -q db-pg18-rehearsal)"
+
+/usr/bin/time -v bun apps/server/scripts/restore-db.ts \
+ --compose-file compose.pg18-rehearsal.yml \
+ --service db-pg18-rehearsal \
+ --input "${RUN_DIR}/pg17-source.dump" \
+ --confirm-empty-target
+docker compose -f compose.pg18-rehearsal.yml exec -T db-pg18-rehearsal \
+ du -sb /var/lib/postgresql
+```
+
+restore は user table だけでなく enum、domain、function、sequence、view、Drizzle
+schema もない target にしか実行できない。途中失敗した target は修復して再利用
+せず、停止後に隔離 volume を破棄し、新しい空 volume で最初からやり直す。
+
+### 2.4 migration、ANALYZE、完全一致検証
+
+PG18 の公開 port は既定で `55432` である。`DB_USER`、`DB_PASSWORD`、
+`DB_DATABASE` は compose と同じ値を、保護された実行環境から渡す。
+
+```bash
+DB_HOST=127.0.0.1 \
+DB_PORT="${PG18_REHEARSAL_PORT:-55432}" \
+bun run --cwd apps/server db:migrate
+
+/usr/bin/time -v bun apps/server/scripts/validate-postgres-rehearsal.ts \
+ --compose-file compose.pg18-rehearsal.yml \
+ --service db-pg18-rehearsal \
+ --expected-major 18 \
+ --expected-vector-version 0.8.5 \
+ --expected-report "${RUN_DIR}/pg17-source.json" \
+ --output "${RUN_DIR}/pg18-target.json"
+```
+
+validator は `ANALYZE` の後、PostgreSQL major、pgvector version、source table の exact
+row count、migration prefix（ID/hash）、source constraint
+（name/type/definition/validated）、invalid constraint 0、rollback される read/write
+probe、vector cosine probe を調べる。migration で target にだけ追加される既定
+allowlist は `media_regions=0` と `ccip_embeddings=0` だけであり、それ以外の追加 table
+または異なる件数は no-go になる。
+
+```bash
+jq -e '
+ .ok == true and
+ (.mismatches | length == 0) and
+ (.tableCounts.media_regions == 0) and
+ (.tableCounts.ccip_embeddings == 0) and
+ (.invalidConstraintCount == 0)
+' \
+ "${RUN_DIR}/pg18-target.json"
+```
+
+target report の全 migration ID/hash と追加 constraint は、rehearsal 対象 commit の
+Drizzle artifact と照合して記録する。既定二 table 以外を意図的に追加する release
+では、review 済みの `--allow-added-table table_name=expected_count` を明示する。暗黙に
+allowlist を広げない。
+
+最後に production build の application を PG18 target へ向けて別 terminal で起動し、
+SSR と DB query を通る `/sources` を確認する。`DB_USER`、`DB_PASSWORD`、
+`DB_DATABASE` も rehearsal compose と同じ値を渡す。
+
+```bash
+bun run --cwd apps/server build
+
+DB_HOST=127.0.0.1 \
+DB_PORT="${PG18_REHEARSAL_PORT:-55432}" \
+NITRO_HOST=127.0.0.1 \
+NITRO_PORT=3100 \
+bun run --cwd apps/server start
+```
+
+別 terminal から次を実行する。
+
+```bash
+curl --fail --show-error --silent http://127.0.0.1:3100/sources \
+ > /dev/null
+```
+
+HTTP success だけでなく、application log に DB connection、migration、query、SSR の
+error がないことを確認する。rehearsal では smoke 後に target を破棄する。本番の
+最初の write 前チェックでは worker と writer を停止したまま、read-only smoke と
+して実行する。
+
+### 2.5 rehearsal の go/no-go
+
+次がすべて満たされたときだけ go とする。
+
+- dump/restore/migration/validation がすべて exit code 0。
+- target は PG18、pgvector `0.8.5`、health check ready。
+- source table/count と migration prefix/constraint が一致し、target-only table は
+ review 済み allowlist の exact count、invalid constraint は 0。
+- vector probe と rollback される read/write probe が成功。
+- target の image tag と digest が記録済みで、本番用の pinned image と一致。
+- dump/restore/validation の最大所要時間と必要容量が本番 window 内に収まる。
+- application の read、代表的な write、job claim、CCIP search の smoke test が成功。
+- JSON report、dump SHA-256、時刻、容量、所要時間、ログが保存済み。
+
+一つでも不明・不一致・未記録なら no-go である。rehearsal target を本番へ昇格
+させず、原因修正後に新しい run ID と空 volume で全手順を繰り返す。
+
+## 3. `jobs` を WAL-logged table にする
+
+この操作は Drizzle の通常 migration から分離されている。`ALTER TABLE ... SET
+LOGGED` は table rewrite と lock を伴うため、承認済み maintenance window で行う。
+
+### 3.1 quiesce と dry-run audit
+
+1. 新規 job の受付を止める。
+2. application server と全 worker instance を止める。
+3. `in_progress` が 0 になるまで待つ。強制停止した job は lease recovery 方針を
+ 記録し、手作業で成功扱いにしない。
+4. PostgreSQL 接続用の `DB_HOST`、`DB_PORT`、`DB_USER`、`DB_PASSWORD`、
+ `DB_DATABASE` を設定して audit を実行する。
+
+```bash
+date --iso-8601=seconds
+docker compose -f compose.yml exec -T db \
+ df -B1 /var/lib/postgresql/data
+bun apps/server/scripts/set-jobs-logged.ts \
+ > "${RUN_DIR}/jobs-set-logged-dry-run.json"
+jq -e '
+ .mode == "dry-run" and
+ .ready == true and
+ .before.inProgressJobs == 0 and
+ .startedAt and .finishedAt and
+ (.elapsedMs >= 0) and
+ (.maxLockWaitMs == 5000) and
+ (.before.tableBytes >= 0) and
+ (.before.indexBytes >= 0) and
+ (.before.totalBytes >= .before.tableBytes)
+' \
+ "${RUN_DIR}/jobs-set-logged-dry-run.json"
+```
+
+`missingQueueNames`、`orphanParents`、`duplicateActiveDedupeKeys`、
+`duplicateRunningConcurrencyKeys`、`invalidRetryRows` もすべて 0 でなければ no-go。
+行を場当たり的に削除せず、migration/backfill または生成側の原因を修正する。
+
+### 3.2 SET LOGGED と commit 境界
+
+```bash
+/usr/bin/time -v bun apps/server/scripts/set-jobs-logged.ts \
+ --apply \
+ --confirm-jobs-quiesced \
+ > "${RUN_DIR}/jobs-set-logged-apply.json"
+jq -s -e '
+ map(select(.mode == "apply")) as $reports |
+ ($reports | length) == 1 and
+ ($reports[0].after.relpersistence == "p") and
+ ($reports[0].startedAt | type == "string") and
+ ($reports[0].finishedAt | type == "string") and
+ ($reports[0].elapsedMs >= 0) and
+ ($reports[0].maxLockWaitMs == 5000) and
+ (($reports[0].changed == false) or ($reports[0].rewriteElapsedMs >= 0)) and
+ ($reports[0].after.tableBytes >= 0) and
+ ($reports[0].after.indexBytes >= 0) and
+ ($reports[0].after.totalBytes >= $reports[0].after.tableBytes)
+' \
+ "${RUN_DIR}/jobs-set-logged-apply.json"
+docker compose -f compose.yml exec -T db \
+ df -B1 /var/lib/postgresql/data
+date --iso-8601=seconds
+```
+
+初回 rewrite では apply report の `changed=true` と `rewriteElapsedMs >= 0` も確認する。
+すでに `relpersistence=p` なら apply を繰り返さず、dry-run と過去の apply 証跡を
+紐付ける。
+
+apply JSON の `startedAt`、`finishedAt`、`elapsedMs`、`rewriteElapsedMs`、
+`maxLockWaitMs` と、before/after の `tableBytes`、`indexBytes`、`totalBytes` を保存する。
+実行前後の volume free bytes とあわせて table rewrite の実容量・所要時間を記録し、
+rehearsal で maintenance window と空き容量の上限を決める。
+
+apply は transaction 内で advisory lock を取得し、`lock_timeout = 5s` を設定して
+audit を再実行した後に rewrite する。lock timeout、audit 違反、SQL error の場合は
+transaction が rollback されるので、worker を停止したまま原因を調べる。
+
+commit 前が安全な rollback boundary である。commit 後に `SET UNLOGGED` へ戻す
+ことを rollback として扱わない。`SET LOGGED` 自体は job の論理内容を変えないため、
+commit 後の障害は table を permanent のまま保持して worker/startup 側を修正する。
+
+### 3.3 restart claim test
+
+1. 管理対象の idempotent な job を通常の API/UI から一件 enqueue する。SQL で
+ payload を直接 insert しない。
+2. worker を一 instance だけ起動する。
+3. その job が `pending` → `in_progress` → terminal state へ一度だけ遷移し、
+ `claim_token`、`claimed_by`、`heartbeat_at`、`attempt_count` が妥当なことを
+ safe jobs API と構造化ログで確認する。
+4. worker を再起動し、未完了 job の lease recovery 後に二重 side effect なしで
+ claim/complete できることを確認する。
+5. AI queue と default queue の双方で一件ずつ確認してから通常 concurrency と
+ job 受付を戻す。
+
+claim が重複する、heartbeat が更新されない、古い claim が完了を commit する、
+同じ `concurrency_key` が同時実行される、parent progress が terminal child の実数と
+一致しない場合は no-go。worker を再停止し、job を手動完了・削除しない。
+
+## 4. CCIP LanceDB → PostgreSQL/pgvector
+
+### 4.1 store mode と順序
+
+`config.json`（または `CONFIG_PATH` の指すファイル）の
+`lancedb.ccipStoreMode` は次の意味を持つ。変更後は server を再起動し、実際の設定と
+read backend をログで確認する。
+
+| mode | read | write | 用途 |
+| --- | --- | --- | --- |
+| `lance` | legacy Lance | legacy Lance | 移行前だけ |
+| `lance-dual-write` | rollback Lance mirror | rollback mirror → PostgreSQL | initial backfill 後、read switch 前 |
+| `postgres-dual-write` | PostgreSQL | PostgreSQL → rollback Lance mirror | read switch 後の観測期間 |
+| `postgres` | PostgreSQL | PostgreSQL | 観測完了後 |
+| `lance-readonly` | rollback Lance | 拒否 | write-freeze 中の緊急調査だけ |
+
+dual-write は同期処理であり、secondary failure を成功として隠さない。部分成功は
+`CcipDualWriteError` として記録される。再試行は key 単位で idempotent でなければ
+ならない。
+
+移行順序は必ず次の通りにする。
+
+1. immutable legacy snapshot と manifest/fingerprint
+2. dry-run
+3. initial backfill と別 directory の rollback mirror 生成
+4. `lance-dual-write`
+5. final delta と全件 parity
+6. `postgres-dual-write` へ read switch
+7. 連続 7 日の observation
+8. `postgres`
+
+### 4.2 immutable snapshot
+
+CCIP extraction と CCIP vector の create/update/delete を quiesce してから snapshot
+を作る。`lancedb.ccipVectorDir` の live directory と、既存の source dump は変更
+しない。snapshot、rollback mirror、checkpoint はそれぞれ別 directory にする。
+
+以下は GNU coreutils を使う例である。snapshot は元 directory の子に作らない。
+
+```bash
+CCIP_LIVE_DIR=""
+CCIP_SNAPSHOT_DIR=""
+CCIP_ROLLBACK_DIR=""
+install -d -m 0700 "${CCIP_SNAPSHOT_DIR}" "${CCIP_ROLLBACK_DIR}"
+
+(
+ cd "${CCIP_LIVE_DIR}"
+ find . -type f -printf '%P\t%s\n' | sort
+) > "${RUN_DIR}/ccip-live.files.tsv"
+(
+ cd "${CCIP_LIVE_DIR}"
+ find . -type f -print0 | sort -z | xargs -0 -r sha256sum
+) > "${RUN_DIR}/ccip-live.manifest.sha256"
+
+cp -a --reflink=auto -- "${CCIP_LIVE_DIR}/." "${CCIP_SNAPSHOT_DIR}/"
+
+(
+ cd "${CCIP_SNAPSHOT_DIR}"
+ find . -type f -printf '%P\t%s\n' | sort
+) > "${RUN_DIR}/ccip-snapshot.files.tsv"
+(
+ cd "${CCIP_SNAPSHOT_DIR}"
+ find . -type f -print0 | sort -z | xargs -0 -r sha256sum
+) > "${RUN_DIR}/ccip-snapshot.manifest.sha256"
+
+cmp "${RUN_DIR}/ccip-live.files.tsv" \
+ "${RUN_DIR}/ccip-snapshot.files.tsv"
+cmp "${RUN_DIR}/ccip-live.manifest.sha256" \
+ "${RUN_DIR}/ccip-snapshot.manifest.sha256"
+sha256sum "${RUN_DIR}/ccip-snapshot.files.tsv" \
+ "${RUN_DIR}/ccip-snapshot.manifest.sha256" \
+ > "${RUN_DIR}/ccip-snapshot.manifest-components.sha256"
+sha256sum "${RUN_DIR}/ccip-snapshot.manifest-components.sha256" \
+ > "${RUN_DIR}/ccip-snapshot.fingerprint.sha256"
+chmod -R a-w "${CCIP_SNAPSHOT_DIR}"
+du -sb "${CCIP_LIVE_DIR}" "${CCIP_SNAPSHOT_DIR}" "${CCIP_ROLLBACK_DIR}"
+```
+
+manifest には相対 path、file bytes、各 file の SHA-256 を辞書順で記録し、その
+manifest 自体の SHA-256 を fingerprint とする。少なくとも次を保存する。
+
+- source directory の canonical path と total bytes
+- snapshot 開始/終了時刻と所要時間
+- file count、manifest path、manifest fingerprint
+- source application version、LanceDB library version、table/schema metadata
+- model、embedding version、vector dimension、logical row count
+
+snapshot 作成後に source と snapshot の manifest fingerprint を再計算し、完全一致
+を確認する。snapshot を read-only にし、その後の migration は snapshot path を
+source とする。dry-run または backfill 後に fingerprint が変化したら no-go。
+
+### 4.3 dry-run、checkpoint、initial backfill
+
+実行時点の正確な引数は script 自身を source of truth とし、最初に help を保存する。
+
+```bash
+bun run --cwd apps/server ccip:migrate-from-lancedb --help \
+ > "${RUN_DIR}/ccip-migrate-help.txt"
+
+/usr/bin/time -v bun run --cwd apps/server ccip:migrate-from-lancedb \
+ --dry-run \
+ --source-dir "${CCIP_SNAPSHOT_DIR}" \
+ --batch-size 100 \
+ --checkpoint "${RUN_DIR}/ccip-dry-run.checkpoint.json" \
+ --report "${RUN_DIR}/ccip-dry-run.json" \
+ --rollback-dir "${CCIP_ROLLBACK_DIR}"
+```
+
+dry-run は PostgreSQL、snapshot、live LanceDB、rollback mirror を変更してはならない。
+manifest/fingerprint、logical key、metadata/revision、vector dimension と finite value、
+参照先 media/region を検証する。次は warning や silent skip ではなく exit code 非 0
+の no-go とする。
+
+- 同じ `(regionId, model, embeddingVersion, preprocessingProfile)` に内容の異なる
+ 複数 record がある
+- media/region の orphan、invalid UUID、invalid dimension、NaN/Infinity がある
+- full region の source/input revision または preprocessing profile を導出できない
+- snapshot manifest/fingerprint が変化した
+
+initial backfill は deterministic key 順、固定 batch size で行う。checkpoint には
+snapshot fingerprint、最後に commit 済みの key、件数、実行 version を atomic に
+保存する。process crash 後は同じ fingerprint の checkpoint だけを resume できる。
+異なる snapshot、code revision、schema で checkpoint を再利用しない。batch commit
+前の key は再処理され得るため、upsert は idempotent でなければならない。
+
+dry-run が go なら、同じ snapshot と rollback directory、および実 migration 専用の
+checkpoint を指定して backfill する。dry-run の checkpoint を実 migration に再利用
+しない。`--resume` は中断された実 migration の checkpoint が存在し fingerprint が
+一致するときだけ付ける。
+
+```bash
+/usr/bin/time -v bun run --cwd apps/server ccip:migrate-from-lancedb \
+ --source-dir "${CCIP_SNAPSHOT_DIR}" \
+ --batch-size 100 \
+ --checkpoint "${RUN_DIR}/ccip-initial.checkpoint.json" \
+ --report "${RUN_DIR}/ccip-initial-backfill.json" \
+ --rollback-dir "${CCIP_ROLLBACK_DIR}"
+```
+
+中断後の再開だけは、同じコマンドへ `--resume` を追加する。
+
+`--source-id` を使う分割実行は rehearsal と原因調査には使えるが、本番 read switch
+の判定は source filter なしの全件 report で行う。実行後、raw rows、unique logical
+rows、collapsed identical duplicates、insert/update 件数を JSON report として保存する。
+
+### 4.4 dual-write、final delta、全件 parity
+
+initial backfill が成功したら `lance-dual-write` で server を再起動し、read が生成済み
+rollback Lance mirror、全 mutation が rollback mirror と PostgreSQL の両方へ同期成功
+することを確認して writer を再開する。legacy live/source dump を dual-write 先に
+しない。その後 writer を短時間 quiesce し、rollback mirror を読み取り source として
+同じ deterministic migration を full scan し、final delta を適用する。immutable
+snapshot と legacy live/source dump はこの処理でも参照・変更しない。
+
+```bash
+/usr/bin/time -v bun run --cwd apps/server ccip:migrate-from-lancedb \
+ --source-dir "${CCIP_ROLLBACK_DIR}" \
+ --batch-size 100 \
+ --checkpoint "${RUN_DIR}/ccip-final.checkpoint.json" \
+ --report "${RUN_DIR}/ccip-final-delta.json" \
+ --rollback-dir "${CCIP_ROLLBACK_DIR}"
+
+bun run --cwd apps/server ccip:migrate-from-lancedb \
+ --verify-only \
+ --source-dir "${CCIP_ROLLBACK_DIR}" \
+ --report "${RUN_DIR}/ccip-full-parity.json"
+```
+
+quiesce を維持したまま、source filter なしで次の全件 parity を取る。
+
+- exact key set: `(regionId, model, embeddingVersion, preprocessingProfile)`
+- exact metadata: media/source/region、region kind、input revision、preprocessing
+ profile、dimension と schema version
+- vector: dimension と finite value が一致し、対応 vector 間の cosine distance
+ `<= 1e-6`
+- search: 固定した anchor/query/filter/top-K 全件で tie group が一致する。同距離 tie
+ 内の順番だけは問わないが、tie group を跨ぐ欠落・追加は不一致とする
+- Rust rerank: 同じ候補集合・同じ query で最終 tie group と score tolerance が一致
+- orphan/conflict/invalid record が 0
+
+parity は exit code 0、JSON の `ok=true`、mismatch 0 が必要である。sample parity だけで
+read switch してはならない。report に snapshot fingerprint、checkpoint、両 backend
+件数、query seed、top-K、tolerance、Rust/AI service version を含める。
+
+### 4.5 read switch と 7 日 observation
+
+全件 parity が成功した時刻を `T0` とし、`postgres-dual-write` へ変更して server を
+再起動する。read が PostgreSQL、write が PostgreSQL → legacy Lance になったことを
+確認して writer を再開する。
+
+`T0` から連続 7 日、毎日同じ時刻帯に次を行う。
+
+1. 管理対象の test media/region で CCIP record を create する。
+2. source/input revision が変わる update/extract を行う。
+3. record を delete する。
+4. 各操作が両 backend で同じ terminal state になったことを確認する。
+5. source filter なしの全件 parity と、固定 query suite の top-K/tie/Rust rerank
+ parity を実行する。
+6. dual-write partial failure、retry、lease recovery、search latency/error rate を確認する。
+
+一件でも mismatch、未解決 partial write、parity 未実施日があれば時計を 0 日へ戻す。
+原因修正と full parity 成功の時刻を新しい `T0` とし、そこから連続 7 日を取り直す。
+7 日完了後、変更承認を得て `postgres` へ切り替える。
+
+### 4.6 CCIP rollback
+
+- read switch 前: writer を止め、`lance-dual-write` から `lance` へ戻す。
+- observation 中: writer を止め、legacy mirror の full parity を確認できた場合だけ
+ `lance-dual-write` へ戻す。PostgreSQL だけに成功した partial write がある場合は、
+ 先に差分を解消する。
+- `lance-readonly` は rollback mirror の調査用で、mutation を拒否する。通常運転の
+ rollback mode にしない。
+- original source dump/snapshot は変更・削除せず、そこへ逆同期しない。rollback 用
+ live/mirror は別 directory とする。
+
+## 5. 本番 PG18 final cutover
+
+この section 固有の rehearsal と承認、および jobs の restart claim test が完了して
+から、別承認済みの window で行う。CCIP read cutover の 7 日 observation は独立した
+gate であり、この PG18 cutover の前提にはしない。timed rehearsal の dump を再利用
+しない。
+
+### 5.1 write-freeze と fresh dump
+
+1. API の mutation、新規 import、file watcher、scheduler、全 job worker、CCIP writer
+ を停止する。PG17 DB は dump のため稼働させる。
+2. `jobs.status='in_progress'` が 0、dual-write partial failure が 0、保留 transaction
+ がないことを証跡化する。
+3. freeze 直後の source report を作る。
+4. fresh custom dump を作り、bytes/SHA-256/所要時間を記録する。
+5. dump 後に source report をもう一度取り、table counts、migrations、constraints が
+ freeze 直後と完全一致することを確認する。
+
+```bash
+bun apps/server/scripts/validate-postgres-rehearsal.ts \
+ --compose-file compose.yml --service db --expected-major 17 \
+ --expect-vector-unavailable \
+ --output "${RUN_DIR}/final-pg17-before-dump.json"
+
+/usr/bin/time -v bun apps/server/scripts/dump-db.ts \
+ --compose-file compose.yml --service db \
+ --output "${RUN_DIR}/final-pg17.dump"
+stat --format '%n %s bytes' "${RUN_DIR}/final-pg17.dump"
+sha256sum "${RUN_DIR}/final-pg17.dump" \
+ > "${RUN_DIR}/final-pg17.dump.sha256"
+
+bun apps/server/scripts/validate-postgres-rehearsal.ts \
+ --compose-file compose.yml --service db --expected-major 17 \
+ --expect-vector-unavailable \
+ --output "${RUN_DIR}/final-pg17-after-dump.json"
+
+jq -S '{tableCounts,migrations,constraints}' \
+ "${RUN_DIR}/final-pg17-before-dump.json" \
+ > "${RUN_DIR}/final-pg17-before-structure.json"
+jq -S '{tableCounts,migrations,constraints}' \
+ "${RUN_DIR}/final-pg17-after-dump.json" \
+ > "${RUN_DIR}/final-pg17-after-structure.json"
+diff -u "${RUN_DIR}/final-pg17-before-structure.json" \
+ "${RUN_DIR}/final-pg17-after-structure.json"
+```
+
+差分があれば write-freeze は成立していないため no-go。dump を破棄扱いにし、writer
+を特定して freeze からやり直す。
+
+### 5.2 fresh PG18 target と go/no-go
+
+本番 target は rehearsal volume ではなく、未使用の新規 volume を使う。承認済み
+本番 compose には rehearsal と同じ pinned PG18 tag+digest、および PG18 用の
+`/var/lib/postgresql` mount が必要である。現在の `compose.yml` はこの条件を満たす
+本番 cutover 定義ではないため、別変更なしに image/volume を切り替えてはならない。
+
+承認済み compose/service を `restore-db.ts` の `--compose-file`/`--service` へ明示し、
+次の順を崩さない。
+
+1. fresh volume の target を起動して health ready を確認。
+2. `--confirm-empty-target` 付きで final custom dump を restore。
+3. final code revision の `db:migrate` を target 接続情報で実行。
+4. validator で `ANALYZE`、exact count/migration/constraint、PG18、pgvector、vector、
+ rollback read/write probe を確認。
+5. PG17 source と PG18 target の双方で、rehearsal と本番の image tag/digest が完全
+ 一致することを再確認。いずれかの image が変わったら timed rehearsal からやり直す。
+6. application の `/sources` readiness、job/CCIP readiness、起動ログを確認。
+
+すべて成功した時だけ go とする。production server の起動 plugin は job worker と
+startup maintenance も開始するため、現実装の app startup を read-only probe と
+みなしてはならない。application を初めて PG18 へ向けて起動する直前を、PG17 へ
+無損失で戻せる最後の rollback boundary とする。起動時刻を「最初の PG18 write の
+可能性がある時刻」として記録し、直後に `/sources` と構造化ログを確認する。
+
+### 5.3 PostgreSQL rollback boundary
+
+**最初の PG18 write より前**なら、application/worker を停止したまま接続先を
+preserved PG17 へ戻せる。PG18 target は調査用に隔離し、再利用しない。
+
+**最初の PG18 write より後**は、PG18 から PG17 への reverse sync を実装していない。
+したがって PG17 へ単純に接続を戻すと、その時刻以降の write を失うため rollback
+ではない。障害時は直ちに全 writer を停止し、PG18 を正として forward recovery
+する。データ損失を伴う PG17 復帰は incident owner とデータ所有者の別承認がない
+限り実行しない。
+
+## 6. 保持と旧 volume の破棄条件
+
+次のすべてを満たすまで、PG17 volume、final dump と SHA-256、legacy Lance source
+dump/snapshot、CCIP rollback mirror、manifest/checkpoint/parity report を保持する。
+
+- PG18 と pgvector のバックアップを取得し、別の空 target への restore test が成功。
+- PostgreSQL/application/job/CCIP の監視期間が完了し、未解決 mismatch・partial write・
+ retry storm・invalid constraint が 0。
+- CCIP は PostgreSQL read で連続 7 日の observation を完了。
+- retention と rollback boundary をデータ所有者が確認。
+- 破棄対象の volume 名、mount、bytes、最終 backup、復旧手順を二者で照合。
+- Issue/変更申請に破棄承認と時刻が記録済み。
+
+破棄するときも `docker compose down -v`、glob、未展開の環境変数は使わない。
+`docker volume inspect ` で対象を解決し、container から未使用である
+ことを確認してから、exact name 一件だけを別承認で削除する。legacy Lance source
+dump は PostgreSQL/PG18 volume の破棄と同時に削除せず、定めた archive retention
+に従う。
+
+## 7. 最終証跡 checklist
+
+- [ ] 実データ cutover の別承認、担当者、window、連絡先
+- [ ] commit SHA と PG17/PG18 の image tag+digest
+- [ ] phase ごとの開始/終了、所要時間、容量
+- [ ] fresh dump bytes、SHA-256、source before/after report
+- [ ] PG18 exact counts/migrations/constraints、vector/readiness report
+- [ ] `jobs` dry-run/apply audit と restart claim test
+- [ ] immutable Lance manifest/fingerprint と unchanged 再検証
+- [ ] dry-run、checkpoint/resume、initial/final migration report
+- [ ] full key/meta/vector/top-K tie/Rust rerank parity
+- [ ] CCIP 連続 7 日の日次 create/update/delete/parity 記録
+- [ ] go/no-go と rollback boundary の署名、最初の PG18 write 時刻
+- [ ] 旧 volume/dump の retention と破棄承認
diff --git a/docs/runbooks/postgresql-18-rehearsal.md b/docs/runbooks/postgresql-18-rehearsal.md
new file mode 100644
index 000000000..706a4b849
--- /dev/null
+++ b/docs/runbooks/postgresql-18-rehearsal.md
@@ -0,0 +1,166 @@
+# PostgreSQL 18 isolated rehearsal
+
+This runbook validates a PostgreSQL 17 to 18 dump/restore without changing the
+default `compose.yml`, its `db` service, or `db-data/`. The rehearsal uses the
+dedicated `db-pg18-rehearsal` service and the named
+`solid-imager-pg18-rehearsal-data` volume mounted at PostgreSQL 18's
+`/var/lib/postgresql` parent directory.
+
+## Safety boundary
+
+- Do not point the rehearsal compose file at `db-data/` or any production bind
+ mount.
+- Do not run `docker compose down -v`; removing any volume requires separate,
+ explicit approval.
+- Restore refuses a target containing user tables, enums, domains, functions,
+ sequences, views, or a Drizzle schema. A failed/partial target must be
+ discarded and recreated as an isolated rehearsal target before retrying.
+- The final default-image/volume cutover is not part of this rehearsal and must
+ be approved as a separate operational change.
+- Record the source and target image tag and the image ID/digest actually used.
+ A floating source tag is not sufficient evidence by itself.
+
+## Rehearsal
+
+Choose new output paths. The commands below never select a backup implicitly and
+never overwrite an existing report or dump.
+
+Before the timed run, record the images resolved by Compose and the running
+PostgreSQL 17 container. The final PostgreSQL 18 change must use the exact same
+target tag and digest as the successful rehearsal.
+
+```bash
+docker compose -f compose.yml config --images
+docker inspect --format '{{.Config.Image}}|{{.Image}}' \
+ "$(docker compose -f compose.yml ps -q db)"
+docker image inspect pgvector/pgvector:pg17 \
+ --format '{{json .RepoDigests}}'
+docker compose -f compose.pg18-rehearsal.yml config --images
+```
+
+1. Record the PostgreSQL 17 source manifest before the dump:
+
+ ```bash
+ bun apps/server/scripts/validate-postgres-rehearsal.ts \
+ --compose-file compose.yml \
+ --service db \
+ --expected-major 17 \
+ --expect-vector-unavailable \
+ --output /tmp/solid-imager-pg17-source.json
+ ```
+
+2. Create an atomic custom-format dump without a TTY:
+
+ ```bash
+ bun apps/server/scripts/dump-db.ts \
+ --compose-file compose.yml \
+ --service db \
+ --output /tmp/solid-imager-pg17-source.dump
+ ```
+
+3. Start only the isolated PostgreSQL 18 service:
+
+ ```bash
+ docker compose -f compose.pg18-rehearsal.yml up -d --wait db-pg18-rehearsal
+ ```
+
+4. Restore into the verified-empty PostgreSQL 18 database:
+
+ ```bash
+ bun apps/server/scripts/restore-db.ts \
+ --compose-file compose.pg18-rehearsal.yml \
+ --service db-pg18-rehearsal \
+ --input /tmp/solid-imager-pg17-source.dump \
+ --confirm-empty-target
+ ```
+
+5. Point Drizzle explicitly at the PostgreSQL 18 rehearsal port and apply all
+ migrations. `DB_USER`, `DB_PASSWORD`, and `DB_DATABASE` must match the
+ rehearsal Compose environment. This applies the target schema after restore
+ and must not be skipped.
+
+ ```bash
+ DB_HOST=127.0.0.1 \
+ DB_PORT="${PG18_REHEARSAL_PORT:-55432}" \
+ bun run --cwd apps/server db:migrate
+ ```
+
+6. Run `ANALYZE`, verify PostgreSQL 18 and pgvector 0.8.5, exercise a rolled
+ back read/write probe and vector query, and compare exact source-table
+ counts, migration hash prefix, and source constraint definitions with the
+ source manifest. Target-only `media_regions` and `ccip_embeddings` are
+ allowed only at their expected count of zero:
+
+ ```bash
+ bun apps/server/scripts/validate-postgres-rehearsal.ts \
+ --compose-file compose.pg18-rehearsal.yml \
+ --service db-pg18-rehearsal \
+ --expected-major 18 \
+ --expected-vector-version 0.8.5 \
+ --expected-report /tmp/solid-imager-pg17-source.json \
+ --output /tmp/solid-imager-pg18-target.json
+ ```
+
+7. Build and start the application against PostgreSQL 18 in a separate
+ terminal. Use the same `DB_USER`, `DB_PASSWORD`, and `DB_DATABASE` as the
+ rehearsal service.
+
+ ```bash
+ bun run --cwd apps/server build
+
+ DB_HOST=127.0.0.1 \
+ DB_PORT="${PG18_REHEARSAL_PORT:-55432}" \
+ NITRO_HOST=127.0.0.1 \
+ NITRO_PORT=3100 \
+ bun run --cwd apps/server start
+ ```
+
+ From another terminal, exercise the `/sources` SSR route, which performs an
+ application-level database read:
+
+ ```bash
+ curl --fail --show-error --silent http://127.0.0.1:3100/sources \
+ > /dev/null
+ ```
+
+ HTTP success is necessary but not sufficient: verify that the application
+ log has no database connection, migration, query, or SSR errors. Keep job
+ inputs quiesced while testing. The production startup plugin starts the job
+ worker and startup maintenance, so application startup is not a read-only
+ probe and must be treated as the first possible PostgreSQL 18 write during a
+ final cutover.
+
+Any non-empty `mismatches` array or non-zero exit status fails the rehearsal.
+Retain both JSON reports with the dump's operational record.
+
+## Jobs WAL maintenance
+
+`jobs` was historically UNLOGGED. Rewriting it is deliberately outside normal
+Drizzle startup migrations. First stop all job workers and run the read-only
+audit, then use the explicit confirmation flag during an approved maintenance
+window:
+
+```bash
+bun apps/server/scripts/set-jobs-logged.ts
+bun apps/server/scripts/set-jobs-logged.ts --apply --confirm-jobs-quiesced
+```
+
+The apply path refuses active jobs or invalid backfill/dedupe state, takes an
+advisory lock, uses a five-second lock timeout, and verifies `relpersistence=p`.
+
+## Separately approved final cutover
+
+The PostgreSQL major-cutover gate and the CCIP read-cutover gate are independent;
+do not combine them in one maintenance window. The approved PostgreSQL window
+must stop writers, create a fresh dump, restore into a new PostgreSQL 18 volume,
+apply migrations, repeat validation and application readiness, and only then
+change the default compose image and volume layout.
+
+Rollback is required for any count/migration/constraint mismatch, invalid
+constraint, vector failure, or application readiness failure.
+Before the first PostgreSQL 18 application write, rollback means returning the
+application to the preserved PostgreSQL 17 service/image recorded during the
+rehearsal. After the first PostgreSQL 18 write, there is no reverse
+synchronization to PostgreSQL 17, so a simple switch back would lose writes and
+is not a rollback. Never reuse a PostgreSQL 17 data directory directly with
+PostgreSQL 18.
diff --git a/packages/application/src/index.ts b/packages/application/src/index.ts
index 076b33c26..572299079 100644
--- a/packages/application/src/index.ts
+++ b/packages/application/src/index.ts
@@ -10,6 +10,7 @@ export type {
ILogger,
IMediaContextProcessor,
IMediaProcessingService,
+ IMediaRegionRenderer,
IMediaService,
IPresetService,
IProjectService,
@@ -22,6 +23,7 @@ export type {
IUserService,
MediaDumpItemWithImageData,
ReadOptions,
+ RenderedMediaRegion,
SearchOptions,
WriteOptions,
} from "./ports";
@@ -38,6 +40,7 @@ export {
createUserService,
MediaProcessingServiceImpl,
MediaQueryService,
+ MediaRegionService,
MediaServiceImpl,
MediaTransferService,
MediaUploadService,
diff --git a/packages/application/src/ports/ccip-vector-store.ts b/packages/application/src/ports/ccip-vector-store.ts
index f0891b161..6c4260142 100644
--- a/packages/application/src/ports/ccip-vector-store.ts
+++ b/packages/application/src/ports/ccip-vector-store.ts
@@ -1,24 +1,34 @@
export type CcipVectorRecord = {
+ regionId: string | null;
+ regionKind: "full" | "person" | "manual";
mediaId: string;
mediaSourceId: string;
vector: number[];
model: string;
embeddingVersion: number;
mediaModifiedAt: Date;
+ inputRevision: string;
+ preprocessingProfile: string;
extractedAt: Date;
};
export type CcipVectorQuery = {
+ regionId?: string;
+ regionKind?: "full" | "person" | "manual";
mediaSourceId?: string;
model?: string;
embeddingVersion?: number;
+ preprocessingProfile?: string;
};
/** Query used by reads that must not mix embedding spaces. */
export type CcipVectorReadQuery = {
+ regionId?: string;
+ regionKind?: "full" | "person" | "manual";
mediaSourceId?: string;
model: string;
embeddingVersion: number;
+ preprocessingProfile: string;
};
export type CcipVectorMetadata = Omit;
@@ -27,7 +37,18 @@ export type CcipVectorCandidate = CcipVectorRecord & {
cosineDistance: number;
};
+export type CcipEmbeddingKey = {
+ regionId: string;
+ model: string;
+ embeddingVersion: number;
+ preprocessingProfile: string;
+};
+
export interface ICcipVectorStore {
+ getByRegion(
+ regionId: string,
+ query: CcipVectorReadQuery,
+ ): Promise;
get(
mediaId: string,
query: CcipVectorReadQuery,
@@ -43,6 +64,8 @@ export interface ICcipVectorStore {
upsert(record: CcipVectorRecord): Promise;
upsertMany(records: CcipVectorRecord[]): Promise;
delete(mediaId: string): Promise;
+ deleteRegion(regionId: string): Promise;
+ deleteEmbedding(key: CcipEmbeddingKey): Promise;
deleteBySource(mediaSourceId: string): Promise;
listMediaIds(query?: CcipVectorQuery): Promise;
list(query?: CcipVectorQuery): Promise;
diff --git a/packages/application/src/ports/index.ts b/packages/application/src/ports/index.ts
index 17f73a521..862790bfe 100644
--- a/packages/application/src/ports/index.ts
+++ b/packages/application/src/ports/index.ts
@@ -16,6 +16,10 @@ export type {
WriteOptions,
} from "./lancedb-dump-service";
export type { IMediaProcessingService } from "./media-processing-service";
+export type {
+ IMediaRegionRenderer,
+ RenderedMediaRegion,
+} from "./media-region-service";
export type {
DeferredActions,
DeferredJob,
diff --git a/packages/application/src/ports/media-region-service.ts b/packages/application/src/ports/media-region-service.ts
new file mode 100644
index 000000000..0338bd70d
--- /dev/null
+++ b/packages/application/src/ports/media-region-service.ts
@@ -0,0 +1,21 @@
+import type { Media } from "@solid-imager/core/domain/media/schemas";
+import type {
+ MediaRegion,
+ MediaRegionRenderProfile,
+} from "@solid-imager/core/domain/media-regions/schemas";
+
+export type RenderedMediaRegion = {
+ bytes: Uint8Array;
+ format: "webp" | "png";
+ width: number;
+ height: number;
+};
+
+export interface IMediaRegionRenderer {
+ readonly version: string;
+ render(
+ media: Media,
+ region: MediaRegion,
+ profile: MediaRegionRenderProfile,
+ ): Promise;
+}
diff --git a/packages/application/src/ports/media-service.ts b/packages/application/src/ports/media-service.ts
index 3f9b614c0..3143eea18 100644
--- a/packages/application/src/ports/media-service.ts
+++ b/packages/application/src/ports/media-service.ts
@@ -26,6 +26,8 @@ import type {
export type DeferredJob = {
mediaId?: string;
sourcePath?: string;
+ targetId?: string;
+ inputRevision?: string;
type:
| "processMedia"
| "downloadImage"
diff --git a/packages/application/src/ports/tagging-service.ts b/packages/application/src/ports/tagging-service.ts
index ad6989f5d..7daf435ca 100644
--- a/packages/application/src/ports/tagging-service.ts
+++ b/packages/application/src/ports/tagging-service.ts
@@ -9,12 +9,13 @@ export interface ITaggingService {
getTagsForMedia(
mediaSourceId: string,
mediaId: string,
- options?: { skipCache?: boolean },
+ options?: { skipCache?: boolean; signal?: AbortSignal },
): Promise;
getCcipFeature(imageBuffer: ArrayBuffer): Promise;
getCcipFeatureForMedia(
mediaSourceId: string,
mediaId: string,
+ signal?: AbortSignal,
): Promise;
getCcipDifference(feature1: number[], feature2: number[]): Promise;
getCcipDistances(
diff --git a/packages/application/src/services/ccip-vector-service.ts b/packages/application/src/services/ccip-vector-service.ts
index 6452221c2..85f059679 100644
--- a/packages/application/src/services/ccip-vector-service.ts
+++ b/packages/application/src/services/ccip-vector-service.ts
@@ -4,6 +4,10 @@ import type {
} from "@solid-imager/core/domain/media/schemas";
import type { IMediaRepository } from "@solid-imager/core/domain/repositories/media-repository";
import type { SourceRepository } from "@solid-imager/core/domain/repositories/source-repository";
+import {
+ createCcipEmbeddingInputRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
import { asyncPool } from "@solid-imager/core/utils/async-pool";
import type {
CcipVectorMetadata,
@@ -15,6 +19,8 @@ import type { ITaggingService } from "../ports/tagging-service";
export const CCIP_MODEL = "ccip-caformer-24-randaug-pruned";
export const CCIP_EMBEDDING_VERSION = 1;
+export const CCIP_PREPROCESSING_PROFILE =
+ "dghs-imgutils-rs/full-image-default/v1";
const MIN_CANDIDATES = 100;
const CANDIDATE_MULTIPLIER = 5;
const MAX_CANDIDATES = 1000;
@@ -34,7 +40,9 @@ export class CcipVectorService {
mediaSourceId: string,
mediaId: string,
force = false,
+ signal?: AbortSignal,
): Promise<{ record: CcipVectorRecord; skipped: boolean }> {
+ signal?.throwIfAborted();
const existing = force
? null
: await this.deps.vectorStore.get(mediaId, this.currentVectorQuery());
@@ -42,8 +50,10 @@ export class CcipVectorService {
mediaSourceId,
mediaId,
existing,
+ signal,
);
if (!result.skipped) {
+ signal?.throwIfAborted();
await this.deps.vectorStore.upsert(result.record);
}
return result;
@@ -54,6 +64,7 @@ export class CcipVectorService {
mediaIds: string[],
force = false,
concurrency = 1,
+ signal?: AbortSignal,
): Promise<
PromiseSettledResult<{
mediaId: string;
@@ -76,6 +87,7 @@ export class CcipVectorService {
mediaSourceId,
mediaId,
existingById.get(mediaId) ?? null,
+ signal,
)),
}));
const records = results.flatMap((result) =>
@@ -83,6 +95,7 @@ export class CcipVectorService {
? [result.value.record]
: [],
);
+ signal?.throwIfAborted();
await this.deps.vectorStore.upsertMany(records);
return results;
}
@@ -91,23 +104,40 @@ export class CcipVectorService {
mediaSourceId: string,
mediaId: string,
existing: CcipVectorRecord | null,
+ signal?: AbortSignal,
): Promise<{ record: CcipVectorRecord; skipped: boolean }> {
+ signal?.throwIfAborted();
const media = await this.requireImage(mediaSourceId, mediaId);
- if (existing && this.isCurrent(existing, media, mediaSourceId)) {
+ const inputRevision = await this.inputRevision(media);
+ if (
+ existing &&
+ (await this.isCurrent(existing, media, mediaSourceId, inputRevision))
+ ) {
return { record: existing, skipped: true };
}
const result = await this.deps.taggingService.getCcipFeatureForMedia(
mediaSourceId,
mediaId,
+ signal,
);
+ signal?.throwIfAborted();
+ const mediaAfterExtraction = await this.requireImage(mediaSourceId, mediaId);
+ const commitRevision = await this.inputRevision(mediaAfterExtraction);
+ if (commitRevision !== inputRevision) {
+ throw new Error("CCIP input changed while the vector was being extracted");
+ }
const record: CcipVectorRecord = {
+ regionId: null,
+ regionKind: "full",
mediaId,
mediaSourceId,
vector: result.feature,
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
mediaModifiedAt: media.modifiedAt,
+ inputRevision,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
extractedAt: new Date(),
};
return { record, skipped: false };
@@ -127,8 +157,16 @@ export class CcipVectorService {
this.currentVectorQuery(),
);
if (!record) return { status: "missing" };
+ const inputRevision = await this.inputRevision(media);
return {
- status: this.isCurrent(record, media, mediaSourceId) ? "ready" : "stale",
+ status: (await this.isCurrent(
+ record,
+ media,
+ mediaSourceId,
+ inputRevision,
+ ))
+ ? "ready"
+ : "stale",
model: record.model,
extractedAt: record.extractedAt,
};
@@ -185,7 +223,7 @@ export class CcipVectorService {
);
if (
!anchor ||
- !this.isCurrent(anchor, anchorMedia, anchorMedia.mediaSourceId)
+ !(await this.isCurrent(anchor, anchorMedia, anchorMedia.mediaSourceId))
) {
throw new Error("CCIP vector is missing or stale for the anchor media");
}
@@ -221,10 +259,17 @@ export class CcipVectorService {
"CCIP similar media lookup completed",
);
const mediaById = new Map(media.map((item) => [item.id, item]));
- const currentCandidates = candidates.filter((candidate) => {
- const item = mediaById.get(candidate.mediaId);
- return item ? this.isCurrent(candidate, item, item.mediaSourceId) : false;
- });
+ const candidateCurrent = await Promise.all(
+ candidates.map(async (candidate) => {
+ const item = mediaById.get(candidate.mediaId);
+ return item
+ ? await this.isCurrent(candidate, item, item.mediaSourceId)
+ : false;
+ }),
+ );
+ const currentCandidates = candidates.filter(
+ (_candidate, index) => candidateCurrent[index],
+ );
if (currentCandidates.length === 0) {
return { media: [], total: 0, scores: [] };
}
@@ -268,25 +313,51 @@ export class CcipVectorService {
};
}
- private isCurrent(
+ private async isCurrent(
record: CcipVectorRecord,
media: Media,
mediaSourceId: string,
- ): boolean {
+ knownRevision?: string,
+ ): Promise {
+ const currentRevision =
+ knownRevision ?? (await this.inputRevision(media));
return (
record.model === CCIP_MODEL &&
record.embeddingVersion === CCIP_EMBEDDING_VERSION &&
+ record.preprocessingProfile === CCIP_PREPROCESSING_PROFILE &&
record.mediaSourceId === mediaSourceId &&
- // A vector extracted after the media's latest modification represents
- // the current file, regardless of LanceDB timestamp serialization.
- record.extractedAt.getTime() >= media.modifiedAt.getTime()
+ (record.inputRevision === currentRevision ||
+ (record.inputRevision === "legacy-unversioned" &&
+ record.mediaModifiedAt.getTime() === media.modifiedAt.getTime()))
);
}
+ private async sourceRevision(media: Media): Promise {
+ return await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
+ }
+
+ private async inputRevision(media: Media): Promise {
+ return await createCcipEmbeddingInputRevision({
+ sourceRevision: await this.sourceRevision(media),
+ model: CCIP_MODEL,
+ embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
+ });
+ }
+
private currentVectorQuery() {
return {
+ regionKind: "full" as const,
model: CCIP_MODEL,
embeddingVersion: CCIP_EMBEDDING_VERSION,
+ preprocessingProfile: CCIP_PREPROCESSING_PROFILE,
};
}
diff --git a/packages/application/src/services/index.ts b/packages/application/src/services/index.ts
index 48618e2b8..3f43394c1 100644
--- a/packages/application/src/services/index.ts
+++ b/packages/application/src/services/index.ts
@@ -12,6 +12,10 @@ export { createIpService } from "./ip-service";
export { createLanceDbDumpService } from "./lancedb-dump-service";
export { MediaProcessingServiceImpl } from "./media-processing-service";
export { MediaQueryService } from "./media-query-service";
+export {
+ computeMediaSourceRevision,
+ MediaRegionService,
+} from "./media-region-service";
export { MediaServiceImpl, validateFileSignature } from "./media-service";
export { MediaTransferService } from "./media-transfer-service";
export { MediaUploadService } from "./media-upload-service";
diff --git a/packages/application/src/services/media-processing-service.ts b/packages/application/src/services/media-processing-service.ts
index 55a63b9a2..39b15dd4c 100644
--- a/packages/application/src/services/media-processing-service.ts
+++ b/packages/application/src/services/media-processing-service.ts
@@ -5,6 +5,7 @@ import type {
Media,
MediaMetadataContext,
} from "@solid-imager/core/domain/media/schemas";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
import type { IAuthorRepository } from "@solid-imager/core/domain/repositories/author-repository";
import type { CharacterRepository } from "@solid-imager/core/domain/repositories/character-repository";
import type { IIpRepository } from "@solid-imager/core/domain/repositories/ip-repository";
@@ -152,9 +153,19 @@ export class MediaProcessingServiceImpl implements IMediaProcessingService {
}
// Step 3: Queue processMedia job
+ const inputRevision = await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
await this.jobRepo.create({
type: "processMedia",
mediaSourceId,
+ targetId: media.id,
+ inputRevision,
payload: {
mediaId: media.id,
sourcePath: basePath,
@@ -206,6 +217,14 @@ export class MediaProcessingServiceImpl implements IMediaProcessingService {
if (!mediaSourceId) {
throw new Error(`Missing mediaSourceId in job ${job.id}`);
}
+ const inputRevision = await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
// Step 1: Metadata extraction
if (payload.skipMetadataExtraction !== true) {
@@ -257,6 +276,8 @@ export class MediaProcessingServiceImpl implements IMediaProcessingService {
await this.jobRepo.create({
type: "auto_tagging",
mediaSourceId,
+ targetId: media.id,
+ inputRevision,
payload: {
mediaId: media.id,
},
@@ -274,6 +295,8 @@ export class MediaProcessingServiceImpl implements IMediaProcessingService {
await this.jobRepo.createIfUnique({
type: "extract_ccip_vector",
mediaSourceId,
+ targetId: media.id,
+ inputRevision,
payload: {
mediaId: media.id,
},
diff --git a/packages/application/src/services/media-region-service.test.ts b/packages/application/src/services/media-region-service.test.ts
new file mode 100644
index 000000000..7967e4167
--- /dev/null
+++ b/packages/application/src/services/media-region-service.test.ts
@@ -0,0 +1,254 @@
+import type { IMediaStorage } from "@solid-imager/core";
+import {
+ MediaRegionRevisionConflictError,
+ ResourceNotFoundError,
+ StaleMediaRegionError,
+} from "@solid-imager/core/domain/errors";
+import type { TransactionManager } from "@solid-imager/core/domain/interfaces/transaction-manager";
+import {
+ createMediaRegionRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
+import type { Media } from "@solid-imager/core/domain/media/schemas";
+import type { MediaRegion } from "@solid-imager/core/domain/media-regions/schemas";
+import type { IMediaRegionRepository } from "@solid-imager/core/domain/repositories/media-region-repository";
+import type { IMediaRepository } from "@solid-imager/core/domain/repositories/media-repository";
+import type { SourceRepository } from "@solid-imager/core/domain/repositories/source-repository";
+import { describe, expect, it, vi } from "vitest";
+import type { IMediaRegionRenderer } from "../ports/media-region-service";
+import {
+ computeMediaSourceRevision,
+ MediaRegionService,
+} from "./media-region-service";
+
+const MEDIA: Media = {
+ id: "10000000-0000-4000-8000-000000000001",
+ mediaSourceId: "20000000-0000-4000-8000-000000000002",
+ filePath: "images/source.png",
+ fileName: "source.png",
+ mediaType: "image",
+ width: 100,
+ height: 200,
+ fileSize: 1234,
+ description: null,
+ createdAt: new Date("2026-01-01T00:00:00.000Z"),
+ modifiedAt: new Date("2026-01-02T00:00:00.000Z"),
+ indexedAt: new Date("2026-01-03T00:00:00.000Z"),
+ status: "active",
+};
+
+async function makeRegion(
+ kind: "full" | "person" | "manual" = "person",
+ sourceRevisionOverride?: string,
+): Promise {
+ const sourceRevision =
+ sourceRevisionOverride ?? (await computeMediaSourceRevision(MEDIA));
+ const bbox =
+ kind === "full" ? null : { x: 0.1, y: 0.2, width: 0.3, height: 0.4 };
+ const regionRevision = await createMediaRegionRevision({
+ sourceRevision,
+ kind,
+ x: bbox?.x ?? null,
+ y: bbox?.y ?? null,
+ width: bbox?.width ?? null,
+ height: bbox?.height ?? null,
+ label: kind === "person" ? "person" : null,
+ detector: kind === "person" ? "detector" : null,
+ detectorModel: kind === "person" ? "model" : null,
+ detectorVersion: kind === "person" ? "1" : null,
+ manualReason: null,
+ });
+ return {
+ id: "30000000-0000-4000-8000-000000000003",
+ mediaId: MEDIA.id,
+ kind,
+ x: bbox?.x ?? null,
+ y: bbox?.y ?? null,
+ width: bbox?.width ?? null,
+ height: bbox?.height ?? null,
+ sourceWidth: MEDIA.width,
+ sourceHeight: MEDIA.height,
+ sourceModifiedAt: MEDIA.modifiedAt,
+ sourceRevision,
+ regionRevision,
+ label: kind === "person" ? "person" : null,
+ manualReason: null,
+ detectionKey: kind === "person" ? "detection-key" : null,
+ detector: kind === "person" ? "detector" : null,
+ detectorModel: kind === "person" ? "model" : null,
+ detectorVersion: kind === "person" ? "1" : null,
+ score: kind === "person" ? 0.9 : null,
+ createdAt: new Date("2026-01-04T00:00:00.000Z"),
+ updatedAt: new Date("2026-01-04T00:00:00.000Z"),
+ };
+}
+
+function setup(region: MediaRegion, rendererVersion = "renderer-v1") {
+ const regionRepository: IMediaRegionRepository = {
+ findByMediaId: vi.fn(async () => [region]),
+ findById: vi.fn(async () => region),
+ create: vi.fn(async () => region),
+ upsertDetected: vi.fn(async () => region),
+ deleteDetectedNotIn: vi.fn(async () => undefined),
+ update: vi.fn(async (_id, _expectedRevision, data) => ({
+ ...region,
+ kind: data.kind ?? region.kind,
+ x: data.bbox?.x ?? region.x,
+ y: data.bbox?.y ?? region.y,
+ width: data.bbox?.width ?? region.width,
+ height: data.bbox?.height ?? region.height,
+ label: data.label === undefined ? region.label : data.label,
+ manualReason:
+ data.manualReason === undefined
+ ? region.manualReason
+ : data.manualReason,
+ detectionKey:
+ data.detectionKey === undefined
+ ? region.detectionKey
+ : data.detectionKey,
+ regionRevision: data.regionRevision,
+ updatedAt: data.updatedAt,
+ })),
+ delete: vi.fn(async () => true),
+ findMaterializedByDerivationKey: vi.fn(async () => null),
+ createMaterialized: vi.fn(async () => MEDIA),
+ };
+ const renderer: IMediaRegionRenderer = {
+ version: rendererVersion,
+ render: vi.fn(async () => ({
+ bytes: new Uint8Array([1, 2, 3]),
+ format: "webp" as const,
+ width: 30,
+ height: 80,
+ })),
+ };
+ const transactionManager: TransactionManager = {
+ transaction: async (callback) => callback(undefined),
+ };
+ const mediaRepository = {
+ findById: vi.fn(async () => MEDIA),
+ } as Partial as IMediaRepository;
+ const sourceRepository = {
+ findAll: vi.fn(async () => []),
+ findById: vi.fn(async () => null),
+ create: vi.fn(async () => {
+ throw new Error("Not used in this test.");
+ }),
+ update: vi.fn(async () => {
+ throw new Error("Not used in this test.");
+ }),
+ delete: vi.fn(async () => undefined),
+ } as SourceRepository;
+ const mediaStorage = {} as IMediaStorage;
+ const service = new MediaRegionService({
+ regionRepository,
+ mediaRepository,
+ sourceRepository,
+ transactionManager,
+ mediaStorage,
+ renderer,
+ });
+ return { regionRepository, renderer, service };
+}
+
+describe("MediaRegionService", () => {
+ it("uses the shared canonical source revision helper", async () => {
+ await expect(computeMediaSourceRevision(MEDIA)).resolves.toBe(
+ await createMediaSourceRevision({
+ mediaId: MEDIA.id,
+ mediaSourceId: MEDIA.mediaSourceId,
+ modifiedAt: MEDIA.modifiedAt,
+ fileSize: MEDIA.fileSize,
+ width: MEDIA.width,
+ height: MEDIA.height,
+ }),
+ );
+ });
+
+ it("keeps full regions out of every public operation", async () => {
+ const full = await makeRegion("full");
+ const { regionRepository, renderer, service } = setup(full);
+
+ await expect(service.list(MEDIA.id)).resolves.toEqual([]);
+ await expect(
+ service.update({
+ regionId: full.id,
+ expectedRevision: full.regionRevision,
+ label: "forbidden",
+ }),
+ ).rejects.toBeInstanceOf(ResourceNotFoundError);
+ await expect(
+ service.delete(full.id, full.regionRevision),
+ ).rejects.toBeInstanceOf(ResourceNotFoundError);
+ await expect(
+ service.render(full.id, full.regionRevision, { transparent: false }),
+ ).rejects.toBeInstanceOf(ResourceNotFoundError);
+ await expect(
+ service.materialize(full.id, full.regionRevision, { transparent: false }),
+ ).rejects.toBeInstanceOf(ResourceNotFoundError);
+ expect(regionRepository.delete).not.toHaveBeenCalled();
+ expect(renderer.render).not.toHaveBeenCalled();
+ });
+
+ it("turns an edited detected region into a manual region", async () => {
+ const detected = await makeRegion("person");
+ const { regionRepository, service } = setup(detected);
+ const updated = await service.update({
+ regionId: detected.id,
+ expectedRevision: detected.regionRevision,
+ bbox: { x: 0.2, y: 0.2, width: 0.2, height: 0.2 },
+ });
+
+ expect(updated.kind).toBe("manual");
+ expect(regionRepository.update).toHaveBeenCalledWith(
+ detected.id,
+ detected.regionRevision,
+ expect.objectContaining({ kind: "manual", detectionKey: null }),
+ );
+ });
+
+ it("rejects stale regions before rendering", async () => {
+ const staleRevision = "a".repeat(64);
+ const stale = await makeRegion("person", staleRevision);
+ const { renderer, service } = setup(stale);
+
+ await expect(
+ service.render(stale.id, stale.regionRevision, { transparent: false }),
+ ).rejects.toBeInstanceOf(StaleMediaRegionError);
+ await expect(
+ service.materialize(stale.id, stale.regionRevision, {
+ transparent: false,
+ }),
+ ).rejects.toBeInstanceOf(StaleMediaRegionError);
+ expect(renderer.render).not.toHaveBeenCalled();
+ });
+
+ it("rejects an outdated optimistic revision", async () => {
+ const region = await makeRegion("person");
+ const { service } = setup(region);
+ await expect(
+ service.render(region.id, "b".repeat(64), { transparent: false }),
+ ).rejects.toBeInstanceOf(MediaRegionRevisionConflictError);
+ });
+
+ it("changes the ETag when the renderer implementation version changes", async () => {
+ const region = await makeRegion("person");
+ const first = setup(region, "renderer-v1");
+ const second = setup(region, "renderer-v2");
+ const firstIdentity = await first.service.getRenderIdentity(
+ region.id,
+ region.regionRevision,
+ { transparent: false },
+ );
+ const secondIdentity = await second.service.getRenderIdentity(
+ region.id,
+ region.regionRevision,
+ { transparent: false },
+ );
+
+ expect(firstIdentity.etag).not.toBe(secondIdentity.etag);
+ expect(firstIdentity.etag).toMatch(/^"[0-9a-f]{64}"$/);
+ expect(first.renderer.render).not.toHaveBeenCalled();
+ expect(second.renderer.render).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/application/src/services/media-region-service.ts b/packages/application/src/services/media-region-service.ts
new file mode 100644
index 000000000..0ea1abd3c
--- /dev/null
+++ b/packages/application/src/services/media-region-service.ts
@@ -0,0 +1,549 @@
+import type { IMediaStorage } from "@solid-imager/core";
+import {
+ MediaRegionRevisionConflictError,
+ ResourceNotFoundError,
+ StaleMediaRegionError,
+ ValidationError,
+} from "@solid-imager/core/domain/errors";
+import type {
+ Transaction,
+ TransactionManager,
+} from "@solid-imager/core/domain/interfaces/transaction-manager";
+import {
+ createMediaRegionRevision,
+ createMediaSourceRevision,
+} from "@solid-imager/core/domain/media/revision";
+import type { Media } from "@solid-imager/core/domain/media/schemas";
+import type {
+ CreateManualMediaRegion,
+ DetectedRegionInput,
+ MaterializedMediaRegion,
+ MediaRegion,
+ MediaRegionBoundingBox,
+ MediaRegionRenderProfile,
+ SafeMediaRegion,
+ UpdateMediaRegion,
+} from "@solid-imager/core/domain/media-regions/schemas";
+import type {
+ IMediaRegionRepository,
+ NewMediaRegion,
+} from "@solid-imager/core/domain/repositories/media-region-repository";
+import type { IMediaRepository } from "@solid-imager/core/domain/repositories/media-repository";
+import type { SourceRepository } from "@solid-imager/core/domain/repositories/source-repository";
+import { localConnectionSchema } from "@solid-imager/core/domain/sources/schemas";
+import type { IMediaRegionRenderer } from "../ports/media-region-service";
+
+const DETECTOR_NAME = "dghs-imgutils-rs";
+const DETECTOR_MODEL = "person-detection";
+const DETECTOR_VERSION = "1";
+const RENDER_PROFILE_VERSION = "crop-v1";
+
+type MediaRegionServiceDependencies = {
+ regionRepository: IMediaRegionRepository;
+ mediaRepository: IMediaRepository;
+ sourceRepository: SourceRepository;
+ transactionManager: TransactionManager;
+ mediaStorage: IMediaStorage;
+ renderer: IMediaRegionRenderer;
+};
+
+export type PersistedDetectionOptions = {
+ mediaId: string;
+ detections: DetectedRegionInput[];
+ detector?: string;
+ detectorModel?: string;
+ detectorVersion?: string;
+};
+
+function bytesToHex(bytes: Uint8Array): string {
+ return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(
+ "",
+ );
+}
+
+async function sha256(value: string): Promise {
+ const digest = await crypto.subtle.digest(
+ "SHA-256",
+ new TextEncoder().encode(value),
+ );
+ return bytesToHex(new Uint8Array(digest));
+}
+
+/** Canonical revision shared by detection, render and stale checks. */
+export function computeMediaSourceRevision(media: Media): Promise {
+ return createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ });
+}
+
+function toSafeMediaRegion(
+ region: MediaRegion,
+ currentSourceRevision: string,
+): SafeMediaRegion {
+ const { detectionKey: omittedDetectionKey, ...safe } = region;
+ void omittedDetectionKey;
+ return {
+ ...safe,
+ stale: region.sourceRevision !== currentSourceRevision,
+ };
+}
+
+function getBoundingBox(region: MediaRegion): MediaRegionBoundingBox {
+ if (
+ region.x === null ||
+ region.y === null ||
+ region.width === null ||
+ region.height === null
+ ) {
+ throw new ValidationError(`Media region ${region.id} has no crop bounds.`);
+ }
+ return {
+ x: region.x,
+ y: region.y,
+ width: region.width,
+ height: region.height,
+ };
+}
+
+function clamp(value: number, minimum: number, maximum: number): number {
+ return Math.min(maximum, Math.max(minimum, value));
+}
+
+function normalizeDetection(
+ detection: DetectedRegionInput,
+ media: Media,
+): {
+ bbox: MediaRegionBoundingBox;
+ integerBox: [number, number, number, number];
+} | null {
+ const x1 = Math.round(clamp(detection.bbox.x1, 0, media.width));
+ const y1 = Math.round(clamp(detection.bbox.y1, 0, media.height));
+ const x2 = Math.round(clamp(detection.bbox.x2, 0, media.width));
+ const y2 = Math.round(clamp(detection.bbox.y2, 0, media.height));
+ if (x2 <= x1 || y2 <= y1) {
+ return null;
+ }
+ return {
+ bbox: {
+ x: x1 / media.width,
+ y: y1 / media.height,
+ width: (x2 - x1) / media.width,
+ height: (y2 - y1) / media.height,
+ },
+ integerBox: [x1, y1, x2, y2],
+ };
+}
+
+function splitFileName(fileName: string): { base: string; extension: string } {
+ const dot = fileName.lastIndexOf(".");
+ if (dot <= 0) {
+ return { base: fileName, extension: "" };
+ }
+ return { base: fileName.slice(0, dot), extension: fileName.slice(dot + 1) };
+}
+
+function getDirectory(filePath: string): string {
+ const slash = Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\"));
+ return slash < 0 ? "" : filePath.slice(0, slash + 1);
+}
+
+export class MediaRegionService {
+ private readonly regionRepository: IMediaRegionRepository;
+ private readonly mediaRepository: IMediaRepository;
+ private readonly sourceRepository: SourceRepository;
+ private readonly transactionManager: TransactionManager;
+ private readonly mediaStorage: IMediaStorage;
+ private readonly renderer: IMediaRegionRenderer;
+
+ constructor(dependencies: MediaRegionServiceDependencies) {
+ this.regionRepository = dependencies.regionRepository;
+ this.mediaRepository = dependencies.mediaRepository;
+ this.sourceRepository = dependencies.sourceRepository;
+ this.transactionManager = dependencies.transactionManager;
+ this.mediaStorage = dependencies.mediaStorage;
+ this.renderer = dependencies.renderer;
+ }
+
+ async list(mediaId: string): Promise {
+ const media = await this.requireMedia(mediaId);
+ const currentRevision = await computeMediaSourceRevision(media);
+ const regions = await this.regionRepository.findByMediaId(mediaId);
+ return regions
+ .filter((region) => region.kind !== "full")
+ .map((region) => toSafeMediaRegion(region, currentRevision));
+ }
+
+ async createManual(input: CreateManualMediaRegion): Promise {
+ const media = await this.requireImage(input.mediaId);
+ const sourceRevision = await computeMediaSourceRevision(media);
+ const label = input.label ?? null;
+ const manualReason = input.manualReason ?? null;
+ const regionRevision = await createMediaRegionRevision({
+ sourceRevision,
+ kind: "manual",
+ x: input.bbox.x,
+ y: input.bbox.y,
+ width: input.bbox.width,
+ height: input.bbox.height,
+ label,
+ detector: null,
+ detectorModel: null,
+ detectorVersion: null,
+ manualReason,
+ });
+ const region = await this.regionRepository.create({
+ mediaId: media.id,
+ kind: "manual",
+ bbox: input.bbox,
+ sourceWidth: media.width,
+ sourceHeight: media.height,
+ sourceModifiedAt: media.modifiedAt,
+ sourceRevision,
+ regionRevision,
+ label,
+ manualReason,
+ detectionKey: null,
+ detector: null,
+ detectorModel: null,
+ detectorVersion: null,
+ score: null,
+ });
+ return toSafeMediaRegion(region, sourceRevision);
+ }
+
+ async update(input: UpdateMediaRegion): Promise {
+ const current = await this.requireRegion(input.regionId);
+ const media = await this.requireImage(current.mediaId);
+ const sourceRevision = await computeMediaSourceRevision(media);
+ if (current.sourceRevision !== sourceRevision) {
+ throw new StaleMediaRegionError(current.id);
+ }
+ const detectedRegionWasEdited =
+ current.kind === "person" &&
+ (input.bbox !== undefined ||
+ input.label !== undefined ||
+ input.manualReason !== undefined);
+ const nextKind = detectedRegionWasEdited ? "manual" : current.kind;
+ const currentBbox = getBoundingBox(current);
+ const nextBbox = input.bbox ?? currentBbox;
+ const nextLabel = input.label === undefined ? current.label : input.label;
+ const nextManualReason =
+ input.manualReason === undefined
+ ? current.manualReason
+ : input.manualReason;
+ const regionRevision = await createMediaRegionRevision({
+ sourceRevision,
+ kind: nextKind,
+ x: nextBbox.x,
+ y: nextBbox.y,
+ width: nextBbox.width,
+ height: nextBbox.height,
+ label: nextLabel,
+ detector: current.detector,
+ detectorModel: current.detectorModel,
+ detectorVersion: current.detectorVersion,
+ manualReason: nextManualReason,
+ });
+
+ const updated = await this.regionRepository.update(
+ current.id,
+ input.expectedRevision,
+ {
+ bbox: input.bbox,
+ kind: nextKind === current.kind ? undefined : nextKind,
+ regionRevision,
+ label: input.label,
+ manualReason: input.manualReason,
+ detectionKey: detectedRegionWasEdited ? null : undefined,
+ updatedAt: new Date(),
+ },
+ );
+ if (!updated) {
+ throw new MediaRegionRevisionConflictError(current.id);
+ }
+ return toSafeMediaRegion(updated, sourceRevision);
+ }
+
+ async delete(regionId: string, expectedRevision: string): Promise {
+ await this.requireRegion(regionId);
+ const deleted = await this.regionRepository.delete(
+ regionId,
+ expectedRevision,
+ );
+ if (!deleted) {
+ const existing = await this.regionRepository.findById(regionId);
+ if (!existing) {
+ throw new ResourceNotFoundError("Media Region", regionId);
+ }
+ throw new MediaRegionRevisionConflictError(regionId);
+ }
+ }
+
+ async persistDetections(
+ options: PersistedDetectionOptions,
+ ): Promise {
+ const media = await this.requireImage(options.mediaId);
+ const sourceRevision = await computeMediaSourceRevision(media);
+ const detector = options.detector ?? DETECTOR_NAME;
+ const detectorModel = options.detectorModel ?? DETECTOR_MODEL;
+ const detectorVersion = options.detectorVersion ?? DETECTOR_VERSION;
+ const pending: NewMediaRegion[] = [];
+
+ for (const detection of options.detections) {
+ const normalized = normalizeDetection(detection, media);
+ if (!normalized) {
+ continue;
+ }
+ const detectionKey = await sha256(
+ JSON.stringify({
+ mediaId: media.id,
+ sourceRevision,
+ detector,
+ detectorModel,
+ detectorVersion,
+ label: detection.label,
+ bbox: normalized.integerBox,
+ }),
+ );
+ pending.push({
+ mediaId: media.id,
+ kind: "person",
+ bbox: normalized.bbox,
+ sourceWidth: media.width,
+ sourceHeight: media.height,
+ sourceModifiedAt: media.modifiedAt,
+ sourceRevision,
+ regionRevision: await createMediaRegionRevision({
+ sourceRevision,
+ kind: "person",
+ x: normalized.bbox.x,
+ y: normalized.bbox.y,
+ width: normalized.bbox.width,
+ height: normalized.bbox.height,
+ label: detection.label,
+ detector,
+ detectorModel,
+ detectorVersion,
+ manualReason: null,
+ }),
+ label: detection.label,
+ manualReason: null,
+ detectionKey,
+ detector,
+ detectorModel,
+ detectorVersion,
+ score: detection.score,
+ });
+ }
+
+ const regions = await this.transactionManager.transaction(
+ async (tx: Transaction) => {
+ const persisted: MediaRegion[] = [];
+ for (const data of pending) {
+ persisted.push(await this.regionRepository.upsertDetected(data, tx));
+ }
+ await this.regionRepository.deleteDetectedNotIn(
+ media.id,
+ pending.flatMap((region) =>
+ region.detectionKey ? [region.detectionKey] : [],
+ ),
+ tx,
+ );
+ return persisted;
+ },
+ );
+
+ return regions.map((region) => toSafeMediaRegion(region, sourceRevision));
+ }
+
+ async render(
+ regionId: string,
+ expectedRevision: string,
+ profile: MediaRegionRenderProfile,
+ ) {
+ const { media, region } = await this.requireCurrentRegion(
+ regionId,
+ expectedRevision,
+ );
+ return this.renderer.render(media, region, profile);
+ }
+
+ async getRenderIdentity(
+ regionId: string,
+ expectedRevision: string,
+ profile: MediaRegionRenderProfile,
+ ): Promise<{ etag: string }> {
+ const { region } = await this.requireCurrentRegion(
+ regionId,
+ expectedRevision,
+ );
+ const digest = await sha256(
+ JSON.stringify({
+ regionId: region.id,
+ regionRevision: region.regionRevision,
+ sourceRevision: region.sourceRevision,
+ profile,
+ profileVersion: RENDER_PROFILE_VERSION,
+ rendererVersion: this.renderer.version,
+ }),
+ );
+ return { etag: `"${digest}"` };
+ }
+
+ async materialize(
+ regionId: string,
+ expectedRevision: string,
+ profile: MediaRegionRenderProfile,
+ ): Promise {
+ const { media, region } = await this.requireCurrentRegion(
+ regionId,
+ expectedRevision,
+ );
+ const derivationKey = await sha256(
+ JSON.stringify({
+ regionId,
+ regionRevision: region.regionRevision,
+ sourceRevision: region.sourceRevision,
+ profile,
+ profileVersion: RENDER_PROFILE_VERSION,
+ rendererVersion: this.renderer.version,
+ }),
+ );
+ const existing =
+ await this.regionRepository.findMaterializedByDerivationKey(
+ derivationKey,
+ );
+ if (existing) {
+ return {
+ regionId,
+ mediaId: existing.id,
+ fileName: existing.fileName,
+ alreadyExisted: true,
+ };
+ }
+
+ const source = await this.sourceRepository.findById(media.mediaSourceId);
+ if (source?.type !== "local") {
+ throw new ValidationError(
+ "Only local media sources support region materialization.",
+ );
+ }
+ const connection = localConnectionSchema.parse(source.connectionInfo);
+ const rendered = await this.renderer.render(media, region, profile);
+ const originalName = splitFileName(media.fileName);
+ const suffix = profile.transparent ? "transparent" : "crop";
+ const outputFileName = `${originalName.base}.region-${region.id.slice(0, 8)}-${derivationKey.slice(0, 8)}-${suffix}.${rendered.format}`;
+ const filePath = `${getDirectory(media.filePath)}${outputFileName}`;
+ const saved = await this.mediaStorage.saveFile(
+ connection.path,
+ {
+ name: outputFileName,
+ arrayBuffer: async () => rendered.bytes,
+ },
+ { filename: filePath, overwrite: true },
+ );
+
+ try {
+ const materialized = await this.regionRepository.createMaterialized({
+ media: {
+ mediaSourceId: media.mediaSourceId,
+ filePath: saved.filePath,
+ fileName: outputFileName,
+ mediaType: "image",
+ width: saved.width,
+ height: saved.height,
+ fileSize: saved.size,
+ description: `Materialized region from ${media.fileName}`,
+ createdAt: saved.createdAt,
+ modifiedAt: saved.modifiedAt,
+ },
+ parentMediaId: media.id,
+ sourceRegionId: region.id,
+ derivationKey,
+ snapshot: {
+ regionId: region.id,
+ regionRevision: region.regionRevision,
+ sourceRevision: region.sourceRevision,
+ bbox: getBoundingBox(region),
+ label: region.label,
+ profile,
+ profileVersion: RENDER_PROFILE_VERSION,
+ rendererVersion: this.renderer.version,
+ },
+ });
+ return {
+ regionId,
+ mediaId: materialized.id,
+ fileName: outputFileName,
+ alreadyExisted: false,
+ };
+ } catch (error) {
+ const winner =
+ await this.regionRepository.findMaterializedByDerivationKey(
+ derivationKey,
+ );
+ if (winner) {
+ return {
+ regionId,
+ mediaId: winner.id,
+ fileName: winner.fileName,
+ alreadyExisted: true,
+ };
+ }
+ await this.mediaStorage
+ .deleteFile(connection.path, saved.filePath)
+ .catch(() => undefined);
+ throw error;
+ }
+ }
+
+ private async requireMedia(mediaId: string): Promise {
+ const media = await this.mediaRepository.findById(mediaId);
+ if (!media) {
+ throw new ResourceNotFoundError("Media", mediaId);
+ }
+ return media;
+ }
+
+ private async requireImage(mediaId: string): Promise {
+ const media = await this.requireMedia(mediaId);
+ if (media.mediaType !== "image" || media.width <= 0 || media.height <= 0) {
+ throw new ValidationError(
+ "Media regions require an image with dimensions.",
+ );
+ }
+ return media;
+ }
+
+ private async requireRegion(
+ regionId: string,
+ ): Promise {
+ const region = await this.regionRepository.findById(regionId);
+ if (!region || region.kind === "full") {
+ throw new ResourceNotFoundError("Media Region", regionId);
+ }
+ return { ...region, kind: region.kind };
+ }
+
+ private async requireCurrentRegion(
+ regionId: string,
+ expectedRevision: string,
+ ): Promise<{ media: Media; region: MediaRegion }> {
+ const region = await this.requireRegion(regionId);
+ if (region.regionRevision !== expectedRevision) {
+ throw new MediaRegionRevisionConflictError(region.id);
+ }
+ const media = await this.requireImage(region.mediaId);
+ const sourceRevision = await computeMediaSourceRevision(media);
+ if (region.sourceRevision !== sourceRevision) {
+ throw new StaleMediaRegionError(region.id);
+ }
+ getBoundingBox(region);
+ return { media, region };
+ }
+}
diff --git a/packages/application/src/services/media-transfer-service.ts b/packages/application/src/services/media-transfer-service.ts
index 1b79a8f55..00322f103 100644
--- a/packages/application/src/services/media-transfer-service.ts
+++ b/packages/application/src/services/media-transfer-service.ts
@@ -1,6 +1,7 @@
import path from "node:path";
import type { IMediaStorage } from "@solid-imager/core";
import { ResourceNotFoundError } from "@solid-imager/core/domain/errors";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
import type {
Transaction,
TransactionManager,
@@ -185,10 +186,20 @@ export class MediaTransferService {
}
const sourcePath = targetConnection.path;
+ const inputRevision = await createMediaSourceRevision({
+ mediaId: newMediaEntry.id,
+ mediaSourceId: newMediaEntry.mediaSourceId,
+ modifiedAt: newMediaEntry.modifiedAt,
+ fileSize: newMediaEntry.fileSize,
+ width: newMediaEntry.width,
+ height: newMediaEntry.height,
+ });
const deferredJob = {
mediaId: newMediaEntry.id,
sourcePath,
type: "processMedia" as const,
+ targetId: newMediaEntry.id,
+ inputRevision,
payload: {
mediaId: newMediaEntry.id,
sourcePath,
@@ -232,6 +243,8 @@ export class MediaTransferService {
await this.jobRepo.create({
type: "processMedia",
mediaSourceId: validatedTargetSourceId,
+ targetId: newMediaEntry.id,
+ inputRevision,
payload: {
mediaId: newMediaEntry.id,
sourcePath,
diff --git a/packages/application/src/services/media-upload-service.ts b/packages/application/src/services/media-upload-service.ts
index 42c4e91e7..c004f42a2 100644
--- a/packages/application/src/services/media-upload-service.ts
+++ b/packages/application/src/services/media-upload-service.ts
@@ -1,6 +1,7 @@
import path from "node:path";
import type { IMediaStorage } from "@solid-imager/core";
import { ResourceNotFoundError } from "@solid-imager/core/domain/errors";
+import { createMediaSourceRevision } from "@solid-imager/core/domain/media/revision";
import {
type AddMediaRequest,
type Media,
@@ -133,9 +134,19 @@ export class MediaUploadService {
]);
}
+ const inputRevision = await createMediaSourceRevision({
+ mediaId: insertedMedia.id,
+ mediaSourceId: insertedMedia.mediaSourceId,
+ modifiedAt: insertedMedia.modifiedAt,
+ fileSize: insertedMedia.fileSize,
+ width: insertedMedia.width,
+ height: insertedMedia.height,
+ });
await this.jobRepo.create({
type: "processMedia",
mediaSourceId: validatedSourceId,
+ targetId: insertedMedia.id,
+ inputRevision,
payload: {
mediaId: insertedMedia.id,
sourcePath: basePath,
@@ -201,9 +212,21 @@ export class MediaUploadService {
if (newMediaItems.length > 0) {
for (const item of newMediaItems) {
+ const media = await this.mediaRepository.findById(item.id);
await this.jobRepo.create({
type: "processMedia",
mediaSourceId: validatedSourceId,
+ targetId: item.id,
+ inputRevision: media
+ ? await createMediaSourceRevision({
+ mediaId: media.id,
+ mediaSourceId: media.mediaSourceId,
+ modifiedAt: media.modifiedAt,
+ fileSize: media.fileSize,
+ width: media.width,
+ height: media.height,
+ })
+ : null,
payload: {
mediaId: item.id,
sourcePath: directoryPath,
diff --git a/packages/application/src/services/tagging-service.ts b/packages/application/src/services/tagging-service.ts
index bc2fa610b..b91d504ea 100644
--- a/packages/application/src/services/tagging-service.ts
+++ b/packages/application/src/services/tagging-service.ts
@@ -61,8 +61,9 @@ export class TaggingServiceImpl implements ITaggingService {
async getTagsForMedia(
mediaSourceId: string,
mediaId: string,
- options?: { skipCache?: boolean },
+ options?: { skipCache?: boolean; signal?: AbortSignal },
): Promise {
+ options?.signal?.throwIfAborted();
const media = await this.mediaRepo.findById(mediaId);
if (!media) {
throw new Error(`Media not found: ${mediaId}`);
@@ -160,14 +161,22 @@ export class TaggingServiceImpl implements ITaggingService {
const canUsePathApi = this.isAiServiceLocal();
if (canUsePathApi) {
- response = await this.aiClient.tagImageByPath(fullPath);
+ response = await this.aiClient.tagImageByPath(fullPath, options?.signal);
} else {
const buffer = await this.readFileBuffer(fullPath);
- response = await this.aiClient.tagImage(buffer);
+ options?.signal?.throwIfAborted();
+ response = await this.aiClient.tagImage(buffer, options?.signal);
}
// Save to DB
- await this.saveTags(mediaSourceId, mediaId, media.filePath, response);
+ options?.signal?.throwIfAborted();
+ await this.saveTags(
+ mediaSourceId,
+ mediaId,
+ media.filePath,
+ response,
+ options?.signal,
+ );
return response;
}
@@ -177,7 +186,9 @@ export class TaggingServiceImpl implements ITaggingService {
mediaId: string,
filePath: string,
response: TaggingResponse,
+ signal?: AbortSignal,
): Promise {
+ signal?.throwIfAborted();
// 1. Tags
const tagsToInsert = Object.entries(response.general).map(
([name, confidence]) => ({
@@ -187,6 +198,7 @@ export class TaggingServiceImpl implements ITaggingService {
}),
);
await this.tagRepo.addTagsToMedia(mediaId, tagsToInsert, "AI");
+ signal?.throwIfAborted();
// 2. IPs — bulk find-or-create
const ipNames = response.ips;
@@ -210,6 +222,7 @@ export class TaggingServiceImpl implements ITaggingService {
if (ipsToLink.length > 0) {
await this.ipRepo.addMediaBulk(mediaId, ipsToLink, "AI");
}
+ signal?.throwIfAborted();
// 3. Characters
// ips_mapping: { charName: [ipName] }
@@ -274,11 +287,13 @@ export class TaggingServiceImpl implements ITaggingService {
bulkCharData,
"AI",
);
+ signal?.throwIfAborted();
// Bulk IP updates for existing characters
if (charsNeedingUpdate.length > 0) {
await this.characterRepo.updateIpsBulk(charsNeedingUpdate, "AI");
}
+ signal?.throwIfAborted();
// Build character link list for addToMediaBulk
const charsToLink: { id: string; confidence: number }[] = [];
@@ -297,6 +312,7 @@ export class TaggingServiceImpl implements ITaggingService {
if (charsToLink.length > 0) {
await this.characterRepo.addToMediaBulk(mediaId, charsToLink, "AI");
}
+ signal?.throwIfAborted();
// Notify clients of the update
this.publishSourceEvent(mediaSourceId, "media-changed", {
@@ -313,7 +329,9 @@ export class TaggingServiceImpl implements ITaggingService {
async getCcipFeatureForMedia(
mediaSourceId: string,
mediaId: string,
+ signal?: AbortSignal,
): Promise {
+ signal?.throwIfAborted();
const media = await this.mediaRepo.findById(mediaId);
if (!media) {
throw new Error(`Media not found: ${mediaId}`);
@@ -344,10 +362,11 @@ export class TaggingServiceImpl implements ITaggingService {
const canUsePathApi = this.isAiServiceLocal();
if (canUsePathApi) {
- return await this.aiClient.extractCcipFeatureByPath(fullPath);
+ return await this.aiClient.extractCcipFeatureByPath(fullPath, signal);
}
const buffer = await this.readFileBuffer(fullPath);
- return await this.aiClient.extractCcipFeature(buffer);
+ signal?.throwIfAborted();
+ return await this.aiClient.extractCcipFeature(buffer, signal);
}
async getCcipDifference(
diff --git a/packages/core/package.json b/packages/core/package.json
index e27169ca7..ac40ebce3 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -12,6 +12,7 @@
"./domain/repositories/*": "./src/domain/repositories/*.ts",
"./domain/services/*": "./src/domain/services/*.ts",
"./domain/interfaces/*": "./src/domain/interfaces/*.ts",
+ "./domain/jobs/*": "./src/domain/jobs/*.ts",
"./domain/search/*": "./src/domain/search/*.ts",
"./domain/sources/*": "./src/domain/sources/*.ts",
"./domain/tagging/*": "./src/domain/tagging/*.ts",
diff --git a/packages/core/src/domain/config/config-schema.ts b/packages/core/src/domain/config/config-schema.ts
index ae4236cc2..f8cb3f3bd 100644
--- a/packages/core/src/domain/config/config-schema.ts
+++ b/packages/core/src/domain/config/config-schema.ts
@@ -155,12 +155,24 @@ export const LanceDbConfigSchema = z.object({
autoFullSync: z.boolean().default(true),
cacheDir: z.string().default(".cache/lancedb-cache"),
ccipVectorDir: z.string().default(".cache/lancedb-ccip"),
+ ccipRollbackDir: z.string().default(".cache/lancedb-ccip-rollback-v1"),
+ ccipStoreMode: z
+ .enum([
+ "lance",
+ "postgres",
+ "postgres-dual-write",
+ "lance-dual-write",
+ "lance-readonly",
+ ])
+ .default("lance"),
});
const DEFAULT_LANCEDB_CONFIG = {
autoFullSync: true,
cacheDir: ".cache/lancedb-cache",
ccipVectorDir: ".cache/lancedb-ccip",
+ ccipRollbackDir: ".cache/lancedb-ccip-rollback-v1",
+ ccipStoreMode: "lance",
} as const;
export const AppConfigSchema = z.object({
diff --git a/packages/core/src/domain/contract/index.ts b/packages/core/src/domain/contract/index.ts
index 1bac24dfc..9958764c1 100644
--- a/packages/core/src/domain/contract/index.ts
+++ b/packages/core/src/domain/contract/index.ts
@@ -9,6 +9,7 @@ export { importsContract } from "./imports.contract";
export { ipsContract } from "./ips.contract";
export { jobsContract } from "./jobs.contract";
export { mediaContract } from "./media.contract";
+export { mediaRegionsContract } from "./media-regions.contract";
export { presetsContract } from "./presets.contract";
export { projectsContract } from "./projects.contract";
export { sourcesContract } from "./sources.contract";
@@ -27,6 +28,7 @@ import { importsContract } from "./imports.contract";
import { ipsContract } from "./ips.contract";
import { jobsContract } from "./jobs.contract";
import { mediaContract } from "./media.contract";
+import { mediaRegionsContract } from "./media-regions.contract";
import { presetsContract } from "./presets.contract";
import { projectsContract } from "./projects.contract";
import { sourcesContract } from "./sources.contract";
@@ -42,6 +44,7 @@ export const appContract = {
sources: sourcesContract,
tags: tagsContract,
media: mediaContract,
+ mediaRegions: mediaRegionsContract,
categories: categoriesContract,
projects: projectsContract,
characters: charactersContract,
diff --git a/packages/core/src/domain/contract/jobs.contract.ts b/packages/core/src/domain/contract/jobs.contract.ts
index bb871f36d..bff7f7a11 100644
--- a/packages/core/src/domain/contract/jobs.contract.ts
+++ b/packages/core/src/domain/contract/jobs.contract.ts
@@ -1,6 +1,11 @@
import { eventIterator, oc } from "@orpc/contract";
+import { z } from "zod";
+import { safeJobSchema } from "../jobs/schemas";
import { jobEventSchema } from "../sources/events";
export const jobsContract = {
+ get: oc
+ .input(z.object({ id: z.string().uuid() }))
+ .output(safeJobSchema.nullable()),
events: oc.output(eventIterator(jobEventSchema)),
};
diff --git a/packages/core/src/domain/contract/media-regions.contract.ts b/packages/core/src/domain/contract/media-regions.contract.ts
new file mode 100644
index 000000000..686b26bfa
--- /dev/null
+++ b/packages/core/src/domain/contract/media-regions.contract.ts
@@ -0,0 +1,26 @@
+import { oc } from "@orpc/contract";
+import { z } from "zod";
+import {
+ createManualMediaRegionSchema,
+ deleteMediaRegionSchema,
+ materializedMediaRegionSchema,
+ materializeMediaRegionSchema,
+ safeMediaRegionSchema,
+ updateMediaRegionSchema,
+} from "../media-regions/schemas";
+
+export const mediaRegionsContract = {
+ list: oc
+ .input(z.object({ mediaId: z.string().uuid() }))
+ .output(z.array(safeMediaRegionSchema)),
+ createManual: oc
+ .input(createManualMediaRegionSchema)
+ .output(safeMediaRegionSchema),
+ update: oc.input(updateMediaRegionSchema).output(safeMediaRegionSchema),
+ delete: oc
+ .input(deleteMediaRegionSchema)
+ .output(z.object({ success: z.literal(true) })),
+ materialize: oc
+ .input(materializeMediaRegionSchema)
+ .output(materializedMediaRegionSchema),
+};
diff --git a/packages/core/src/domain/errors/index.ts b/packages/core/src/domain/errors/index.ts
index 962361e94..dfbf21da6 100644
--- a/packages/core/src/domain/errors/index.ts
+++ b/packages/core/src/domain/errors/index.ts
@@ -22,6 +22,20 @@ export class ResourceNotFoundError extends DomainError {
export class ResourceConflictError extends DomainError {}
+export class MediaRegionRevisionConflictError extends ResourceConflictError {
+ constructor(regionId: string) {
+ super(`Media region ${regionId} was changed by another request.`);
+ }
+}
+
+export class StaleMediaRegionError extends ResourceConflictError {
+ constructor(regionId: string) {
+ super(
+ `Media region ${regionId} no longer matches the current source media revision.`,
+ );
+ }
+}
+
export class ValidationError extends DomainError {}
export class UnexpectedError extends DomainError {
diff --git a/packages/core/src/domain/interfaces/ai-client.ts b/packages/core/src/domain/interfaces/ai-client.ts
index 502980659..e8cd811af 100644
--- a/packages/core/src/domain/interfaces/ai-client.ts
+++ b/packages/core/src/domain/interfaces/ai-client.ts
@@ -8,15 +8,21 @@ import type {
export type IAiClient = {
healthCheck(): Promise;
- tagImage(imageBuffer: ArrayBuffer): Promise;
+ tagImage(imageBuffer: ArrayBuffer, signal?: AbortSignal): Promise;
- tagImageByPath(path: string): Promise;
+ tagImageByPath(path: string, signal?: AbortSignal): Promise;
tagImageOppaiOracleByPath(path: string): Promise;
- extractCcipFeature(imageBuffer: ArrayBuffer): Promise;
+ extractCcipFeature(
+ imageBuffer: ArrayBuffer,
+ signal?: AbortSignal,
+ ): Promise;
- extractCcipFeatureByPath(path: string): Promise;
+ extractCcipFeatureByPath(
+ path: string,
+ signal?: AbortSignal,
+ ): Promise;
calculateCcipDifference(
feature1: number[],
diff --git a/packages/core/src/domain/jobs/registry.test.ts b/packages/core/src/domain/jobs/registry.test.ts
new file mode 100644
index 000000000..502a57150
--- /dev/null
+++ b/packages/core/src/domain/jobs/registry.test.ts
@@ -0,0 +1,151 @@
+import { describe, expect, it } from 'vite-plus/test';
+import { prepareJob, retryDelayMs, validateJobPayload } from './registry';
+import { JOB_TYPES } from './schemas';
+
+const MEDIA_ID = '11111111-1111-4111-8111-111111111111';
+const SOURCE_ID = '22222222-2222-4222-8222-222222222222';
+const PARENT_ID = '33333333-3333-4333-8333-333333333333';
+
+describe('job registry', () => {
+ it('defines a runtime payload schema for every durable job type', () => {
+ const payloads: Record<(typeof JOB_TYPES)[number], unknown> = {
+ processMedia: { mediaId: MEDIA_ID, sourcePath: '/media' },
+ downloadImage: { targetUrl: 'https://example.com/image.png' },
+ auto_tagging: { mediaId: MEDIA_ID },
+ extract_ccip_vector: { mediaId: MEDIA_ID },
+ bulk_tagging_parent: { total: 0, processed: 0, failed: 0 },
+ bulk_tagging_dispatch: { mediaSourceId: SOURCE_ID },
+ batch_ccip_parent: { total: 0, processed: 0, failed: 0 },
+ batch_ccip_dispatch: { mediaSourceId: SOURCE_ID },
+ import_request: { targetUrl: 'https://example.com/image.png' },
+ sync_lancedb: null,
+ sync_lancedb_full: { reason: 'test' },
+ sync_lancedb_delta: { mediaIds: [MEDIA_ID] },
+ };
+
+ for (const type of JOB_TYPES) {
+ expect(validateJobPayload(type, payloads[type]).success).toBe(true);
+ }
+ });
+
+ it('assigns deterministic queue, retry, dedupe and concurrency policy', () => {
+ const prepared = prepareJob({
+ type: 'auto_tagging',
+ mediaSourceId: SOURCE_ID,
+ targetId: MEDIA_ID,
+ inputRevision: 'revision-1',
+ payload: { mediaId: MEDIA_ID, force: true },
+ });
+
+ expect(prepared.queueName).toBe('ai');
+ expect(prepared.maxAttempts).toBe(5);
+ expect(prepared.leaseDurationMs).toBe(300_000);
+ expect(prepared.dedupeKey).toBe(
+ `auto_tagging:${MEDIA_ID}:revision-1:force`,
+ );
+ expect(prepared.concurrencyKey).toBe(`media:${MEDIA_ID}:auto_tagging`);
+ });
+
+ it('scopes batch child dedupe to its parent', () => {
+ const prepared = prepareJob({
+ type: 'extract_ccip_vector',
+ mediaSourceId: SOURCE_ID,
+ parentId: PARENT_ID,
+ targetId: MEDIA_ID,
+ inputRevision: 'revision-2',
+ payload: { mediaId: MEDIA_ID },
+ });
+
+ expect(prepared.dedupeKey).toBe(
+ `extract_ccip_vector:${PARENT_ID}:${MEDIA_ID}:revision-2:normal`,
+ );
+ });
+
+ it('keeps full and metadata-skip processing requests distinct but serialized', () => {
+ const full = prepareJob({
+ type: 'processMedia',
+ mediaSourceId: SOURCE_ID,
+ targetId: MEDIA_ID,
+ inputRevision: 'revision-3',
+ payload: { mediaId: MEDIA_ID, sourcePath: '/media' },
+ });
+ const skip = prepareJob({
+ type: 'processMedia',
+ mediaSourceId: SOURCE_ID,
+ targetId: MEDIA_ID,
+ inputRevision: 'revision-3',
+ payload: {
+ mediaId: MEDIA_ID,
+ sourcePath: '/media',
+ skipMetadataExtraction: true,
+ },
+ });
+
+ expect(full.dedupeKey).toContain('metadata-full');
+ expect(skip.dedupeKey).toContain('metadata-skip');
+ expect(full.dedupeKey).not.toBe(skip.dedupeKey);
+ expect(full.concurrencyKey).toBe(skip.concurrencyKey);
+ });
+
+ it('keeps force requests distinct while serializing each AI target', () => {
+ for (const type of ['auto_tagging', 'extract_ccip_vector'] as const) {
+ const normal = prepareJob({
+ type,
+ mediaSourceId: SOURCE_ID,
+ targetId: MEDIA_ID,
+ inputRevision: 'revision-4',
+ payload: { mediaId: MEDIA_ID, force: false },
+ });
+ const force = prepareJob({
+ type,
+ mediaSourceId: SOURCE_ID,
+ targetId: MEDIA_ID,
+ inputRevision: 'revision-4',
+ payload: { mediaId: MEDIA_ID, force: true },
+ });
+ expect(normal.dedupeKey).not.toBe(force.dedupeKey);
+ expect(normal.concurrencyKey).toBe(force.concurrencyKey);
+ }
+ });
+
+ it('scopes download dedupe to its destination and leaves import inbox items unique', () => {
+ const url = 'https://example.com/image.png';
+ const first = prepareJob({
+ type: 'downloadImage',
+ mediaSourceId: SOURCE_ID,
+ payload: { targetUrl: url, fileName: 'first.png' },
+ });
+ const second = prepareJob({
+ type: 'downloadImage',
+ mediaSourceId: '44444444-4444-4444-8444-444444444444',
+ payload: { targetUrl: url, fileName: 'first.png' },
+ });
+ const renamed = prepareJob({
+ type: 'downloadImage',
+ mediaSourceId: SOURCE_ID,
+ payload: { targetUrl: url, fileName: 'second.png' },
+ });
+ const importRequest = prepareJob({
+ type: 'import_request',
+ payload: { targetUrl: url },
+ });
+
+ expect(first.dedupeKey).not.toBe(second.dedupeKey);
+ expect(first.dedupeKey).not.toBe(renamed.dedupeKey);
+ expect(importRequest.dedupeKey).toBeNull();
+ });
+
+ it('uses bounded exponential retry delay with deterministic jitter', () => {
+ expect(retryDelayMs(1, 0.5)).toBe(5_000);
+ expect(retryDelayMs(2, 0.5)).toBe(10_000);
+ expect(retryDelayMs(99, 0.5)).toBe(900_000);
+ });
+
+ it('rejects unknown types at creation and invalid payloads at execution', () => {
+ expect(() => prepareJob({ type: 'unknown' })).toThrow('Unknown job type');
+ expect(validateJobPayload('auto_tagging', { mediaId: 'not-a-uuid' }).success).toBe(
+ false,
+ );
+ expect(validateJobPayload('unknown', {}).success).toBe(false);
+ });
+});
diff --git a/packages/core/src/domain/jobs/registry.ts b/packages/core/src/domain/jobs/registry.ts
new file mode 100644
index 000000000..1a4911609
--- /dev/null
+++ b/packages/core/src/domain/jobs/registry.ts
@@ -0,0 +1,179 @@
+import type { NewJob } from "../repositories/job-repository";
+import { jobEnvelopeSchema, type JobQueueName, type JobType } from "./schemas";
+
+export const DEFAULT_JOB_LEASE_MS = 5 * 60 * 1000;
+export const JOB_HEARTBEAT_MS = 30 * 1000;
+
+export type JobPolicy = {
+ queueName: JobQueueName;
+ maxAttempts: number;
+ leaseDurationMs: number;
+};
+
+const DEFAULT_POLICY: JobPolicy = {
+ queueName: "default",
+ maxAttempts: 5,
+ leaseDurationMs: DEFAULT_JOB_LEASE_MS,
+};
+
+const AI_POLICY: JobPolicy = {
+ queueName: "ai",
+ maxAttempts: 5,
+ leaseDurationMs: DEFAULT_JOB_LEASE_MS,
+};
+
+const DISPATCH_POLICY: JobPolicy = {
+ queueName: "default",
+ maxAttempts: 3,
+ leaseDurationMs: DEFAULT_JOB_LEASE_MS,
+};
+
+export const JOB_POLICIES: Record = {
+ processMedia: DEFAULT_POLICY,
+ downloadImage: DEFAULT_POLICY,
+ auto_tagging: AI_POLICY,
+ extract_ccip_vector: AI_POLICY,
+ bulk_tagging_parent: DISPATCH_POLICY,
+ bulk_tagging_dispatch: DISPATCH_POLICY,
+ batch_ccip_parent: DISPATCH_POLICY,
+ batch_ccip_dispatch: DISPATCH_POLICY,
+ import_request: DEFAULT_POLICY,
+ sync_lancedb: DEFAULT_POLICY,
+ sync_lancedb_full: DEFAULT_POLICY,
+ sync_lancedb_delta: DEFAULT_POLICY,
+};
+
+export type PreparedJob = NewJob & {
+ type: JobType;
+ queueName: JobQueueName;
+ maxAttempts: number;
+ leaseDurationMs: number;
+};
+
+export function isKnownJobType(type: string): type is JobType {
+ return Object.hasOwn(JOB_POLICIES, type);
+}
+
+export function validateJobPayload(type: string, payload: unknown) {
+ return jobEnvelopeSchema.safeParse({ type, payload });
+}
+
+/** Adds deterministic queue, retry and fencing metadata at the repository edge. */
+export function prepareJob(job: NewJob): PreparedJob {
+ if (!isKnownJobType(job.type)) {
+ throw new Error(`Unknown job type: ${job.type}`);
+ }
+ const policy = JOB_POLICIES[job.type];
+ const targetId = job.targetId ?? inferTargetId(job);
+ const inputRevision = job.inputRevision ?? inferInputRevision(job);
+ const force = getBoolean(job.payload, "force") ? "force" : "normal";
+ const dedupeKey = job.dedupeKey ?? buildDedupeKey(job.type, targetId, inputRevision, force, job);
+ const concurrencyKey = job.concurrencyKey ?? buildConcurrencyKey(job.type, targetId, job);
+
+ return {
+ ...job,
+ type: job.type,
+ queueName: job.queueName ?? policy.queueName,
+ targetId,
+ inputRevision,
+ dedupeKey,
+ concurrencyKey,
+ availableAt: job.availableAt ?? new Date(),
+ attemptCount: job.attemptCount ?? 0,
+ maxAttempts: job.maxAttempts ?? policy.maxAttempts,
+ leaseDurationMs: job.leaseDurationMs ?? policy.leaseDurationMs,
+ };
+}
+
+export function retryDelayMs(attemptCount: number, random = Math.random()): number {
+ const exponent = Math.max(0, attemptCount - 1);
+ const base = Math.min(15 * 60 * 1000, 5_000 * 2 ** exponent);
+ return Math.min(15 * 60 * 1000, Math.round(base * (0.8 + random * 0.4)));
+}
+
+function inferTargetId(job: NewJob): string | null {
+ const mediaId = getString(job.payload, "mediaId");
+ if (mediaId) return mediaId;
+ if (job.parentId) return job.parentId;
+ return job.mediaSourceId ?? null;
+}
+
+function inferInputRevision(job: NewJob): string | null {
+ return (
+ getString(job.payload, "inputRevision") ?? getString(job.payload, "sourceRevision") ?? null
+ );
+}
+
+function buildDedupeKey(
+ type: JobType,
+ targetId: string | null,
+ inputRevision: string | null,
+ force: string,
+ job: NewJob,
+): string | null {
+ if (type === "import_request") {
+ // Import requests are an inbox: identical URLs may represent separate user
+ // actions and do not have a destination until they are accepted.
+ return null;
+ }
+ if (type === "downloadImage") {
+ const url = getString(job.payload, "targetUrl") ?? getString(job.payload, "imageUrl");
+ const destination =
+ getString(job.payload, "filePath") ?? getString(job.payload, "fileName") ?? "auto";
+ return url && job.mediaSourceId
+ ? `${type}:${job.mediaSourceId}:${destination}:${url}`
+ : null;
+ }
+ if (type === "sync_lancedb" || type === "sync_lancedb_full") {
+ return job.mediaSourceId ? `sync_lancedb_full:${job.mediaSourceId}` : null;
+ }
+ if (type === "sync_lancedb_delta") {
+ return job.mediaSourceId ? `sync_lancedb_delta:${job.mediaSourceId}` : null;
+ }
+ if (type === "bulk_tagging_dispatch" || type === "batch_ccip_dispatch") {
+ return job.parentId ? `${type}:${job.parentId}` : null;
+ }
+ if (type === "bulk_tagging_parent" || type === "batch_ccip_parent") {
+ return null;
+ }
+ if (type === "processMedia") {
+ const mode = getBoolean(job.payload, "skipMetadataExtraction")
+ ? "metadata-skip"
+ : "metadata-full";
+ return targetId
+ ? `${type}:${targetId}:${inputRevision ?? "current"}:${mode}`
+ : null;
+ }
+ if (
+ job.parentId &&
+ (type === "auto_tagging" || type === "extract_ccip_vector") &&
+ targetId
+ ) {
+ return `${type}:${job.parentId}:${targetId}:${inputRevision ?? "current"}:${force}`;
+ }
+ return targetId ? `${type}:${targetId}:${inputRevision ?? "current"}:${force}` : null;
+}
+
+function buildConcurrencyKey(type: JobType, targetId: string | null, job: NewJob): string | null {
+ if (type === "sync_lancedb" || type === "sync_lancedb_full" || type === "sync_lancedb_delta") {
+ return job.mediaSourceId ? `lancedb:${job.mediaSourceId}` : null;
+ }
+ if (type === "auto_tagging" || type === "extract_ccip_vector" || type === "processMedia") {
+ return targetId ? `media:${targetId}:${type}` : null;
+ }
+ return null;
+}
+
+function getString(payload: unknown, key: string): string | null {
+ if (!isRecord(payload)) return null;
+ const value = payload[key];
+ return typeof value === "string" ? value : null;
+}
+
+function getBoolean(payload: unknown, key: string): boolean {
+ return isRecord(payload) && payload[key] === true;
+}
+
+function isRecord(value: unknown): value is Record {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
diff --git a/packages/core/src/domain/jobs/schemas.ts b/packages/core/src/domain/jobs/schemas.ts
new file mode 100644
index 000000000..753ff3fd7
--- /dev/null
+++ b/packages/core/src/domain/jobs/schemas.ts
@@ -0,0 +1,178 @@
+import { z } from "zod";
+import { downloadItemSchema } from "../media/schemas";
+import { batchParentPayloadSchema } from "../tagging/schemas";
+
+export const JOB_TYPES = [
+ "processMedia",
+ "downloadImage",
+ "auto_tagging",
+ "extract_ccip_vector",
+ "bulk_tagging_parent",
+ "bulk_tagging_dispatch",
+ "batch_ccip_parent",
+ "batch_ccip_dispatch",
+ "import_request",
+ "sync_lancedb",
+ "sync_lancedb_full",
+ "sync_lancedb_delta",
+] as const;
+
+export const jobTypeSchema = z.enum(JOB_TYPES);
+export type JobType = z.infer;
+
+export const jobStatusSchema = z.enum([
+ "pending",
+ "in_progress",
+ "completed",
+ "failed",
+ "cancelled",
+]);
+export type JobStatus = z.infer;
+
+export const jobQueueNameSchema = z.enum(["default", "ai"]);
+export type JobQueueName = z.infer;
+
+const processMediaPayloadSchema = z.object({
+ mediaId: z.string().uuid(),
+ sourcePath: z.string().min(1),
+ type: z.literal("processMedia").optional(),
+ skipMetadataExtraction: z.boolean().optional(),
+});
+
+const downloadImagePayloadSchema = downloadItemSchema
+ .extend({
+ imageUrl: z.string().url().optional(),
+ sourceUrl: z.string().url().optional(),
+ })
+ .refine(
+ (payload) => typeof payload.targetUrl === "string" || typeof payload.imageUrl === "string",
+ { message: "downloadImage requires targetUrl or imageUrl" },
+ );
+
+const autoTaggingPayloadSchema = z.object({
+ mediaId: z.string().uuid(),
+ force: z.boolean().optional(),
+});
+
+const singleCcipPayloadSchema = z.object({
+ mediaId: z.string().uuid(),
+ force: z.boolean().optional().default(false),
+});
+
+const batchCcipPayloadSchema = z.object({
+ mediaIds: z.array(z.string().uuid()).min(1).max(25),
+ force: z.boolean().optional().default(false),
+});
+
+const ccipPayloadSchema = z.union([singleCcipPayloadSchema, batchCcipPayloadSchema]);
+
+const dispatchPayloadSchema = z.object({
+ force: z.boolean().optional().default(false),
+ batchSize: z.number().int().positive().max(5000).optional(),
+ mediaSourceId: z.string().uuid().optional(),
+});
+
+const fullSyncPayloadSchema = z
+ .object({
+ reason: z.string().optional(),
+ batchSize: z.number().int().positive().optional(),
+ delayMs: z.number().int().nonnegative().optional(),
+ })
+ .optional()
+ .nullable();
+
+const deltaSyncPayloadSchema = z.object({
+ reason: z.string().optional(),
+ batchSize: z.number().int().positive().optional(),
+ mediaId: z.string().uuid().optional(),
+ mediaIds: z.array(z.string().uuid()).optional(),
+ operation: z.enum(["upsert", "delete"]).optional(),
+});
+
+/**
+ * Runtime source of truth for every durable job payload.
+ *
+ * The database intentionally keeps `jobs.type` as text so a worker can mark an
+ * unknown legacy row as a non-retryable failure instead of failing to read it.
+ */
+export const jobEnvelopeSchema = z.discriminatedUnion("type", [
+ z.object({ type: z.literal("processMedia"), payload: processMediaPayloadSchema }),
+ z.object({
+ type: z.literal("downloadImage"),
+ payload: downloadImagePayloadSchema,
+ }),
+ z.object({
+ type: z.literal("auto_tagging"),
+ payload: autoTaggingPayloadSchema,
+ }),
+ z.object({
+ type: z.literal("extract_ccip_vector"),
+ payload: ccipPayloadSchema,
+ }),
+ z.object({
+ type: z.literal("bulk_tagging_parent"),
+ payload: batchParentPayloadSchema,
+ }),
+ z.object({
+ type: z.literal("bulk_tagging_dispatch"),
+ payload: dispatchPayloadSchema,
+ }),
+ z.object({
+ type: z.literal("batch_ccip_parent"),
+ payload: batchParentPayloadSchema,
+ }),
+ z.object({
+ type: z.literal("batch_ccip_dispatch"),
+ payload: dispatchPayloadSchema,
+ }),
+ z.object({
+ type: z.literal("import_request"),
+ payload: downloadItemSchema,
+ }),
+ z.object({ type: z.literal("sync_lancedb"), payload: fullSyncPayloadSchema }),
+ z.object({
+ type: z.literal("sync_lancedb_full"),
+ payload: fullSyncPayloadSchema,
+ }),
+ z.object({
+ type: z.literal("sync_lancedb_delta"),
+ payload: deltaSyncPayloadSchema,
+ }),
+]);
+
+export type JobEnvelope = z.infer;
+
+export const safeJobProgressSchema = z.object({
+ processed: z.number().int().nonnegative(),
+ failed: z.number().int().nonnegative(),
+ total: z.number().int().nonnegative(),
+});
+
+export const safeJobSchema = z.object({
+ id: z.string().uuid(),
+ type: z.string(),
+ status: jobStatusSchema,
+ queueName: jobQueueNameSchema.nullable(),
+ targetId: z.string().nullable(),
+ inputRevision: z.string().nullable(),
+ attemptCount: z.number().int().nonnegative(),
+ maxAttempts: z.number().int().positive(),
+ errorCode: z.string().nullable(),
+ errorMessage: z.string().nullable(),
+ progress: safeJobProgressSchema.nullable(),
+ createdAt: z.coerce.date(),
+ updatedAt: z.coerce.date(),
+ parentId: z.string().uuid().nullable(),
+});
+export type SafeJob = z.infer;
+
+export function getSafeJobErrorMessage(errorCode: string | null): string | null {
+ if (!errorCode) return null;
+ if (errorCode === "INVALID_JOB_PAYLOAD") return "The job payload is invalid.";
+ if (errorCode === "UNKNOWN_JOB_TYPE") return "The job type is not supported.";
+ if (errorCode === "STALE_INPUT") return "The job input is no longer current.";
+ if (errorCode === "TARGET_NOT_FOUND") return "The job target no longer exists.";
+ if (errorCode === "LEASE_EXPIRED") return "The job worker lease expired.";
+ if (errorCode === "DISPATCH_FAILED") return "The batch dispatcher failed.";
+ return "The job failed.";
+}
diff --git a/packages/core/src/domain/media-regions/schemas.ts b/packages/core/src/domain/media-regions/schemas.ts
new file mode 100644
index 000000000..9828f97b0
--- /dev/null
+++ b/packages/core/src/domain/media-regions/schemas.ts
@@ -0,0 +1,140 @@
+import { z } from "zod";
+
+export const mediaRegionKindSchema = z.enum(["full", "person", "manual"]);
+export type MediaRegionKind = z.infer;
+
+export const mediaRevisionSchema = z
+ .string()
+ .regex(/^[0-9a-f]{64}$/, "Revision must be a lowercase SHA-256 digest");
+
+export const mediaRegionBoundingBoxSchema = z
+ .object({
+ x: z.number().min(0).max(1),
+ y: z.number().min(0).max(1),
+ width: z.number().positive().max(1),
+ height: z.number().positive().max(1),
+ })
+ .refine((bbox) => bbox.x + bbox.width <= 1, {
+ message: "Region must fit within the source width",
+ path: ["width"],
+ })
+ .refine((bbox) => bbox.y + bbox.height <= 1, {
+ message: "Region must fit within the source height",
+ path: ["height"],
+ });
+
+export type MediaRegionBoundingBox = z.infer<
+ typeof mediaRegionBoundingBoxSchema
+>;
+
+export const mediaRegionSchema = z.object({
+ id: z.string().uuid(),
+ mediaId: z.string().uuid(),
+ kind: mediaRegionKindSchema,
+ x: z.number().nullable(),
+ y: z.number().nullable(),
+ width: z.number().nullable(),
+ height: z.number().nullable(),
+ sourceWidth: z.number().int().positive(),
+ sourceHeight: z.number().int().positive(),
+ sourceModifiedAt: z.coerce.date(),
+ sourceRevision: mediaRevisionSchema,
+ regionRevision: mediaRevisionSchema,
+ label: z.string().nullable(),
+ manualReason: z.string().nullable(),
+ detectionKey: z.string().nullable(),
+ detector: z.string().nullable(),
+ detectorModel: z.string().nullable(),
+ detectorVersion: z.string().nullable(),
+ score: z.number().min(0).max(1).nullable(),
+ createdAt: z.coerce.date(),
+ updatedAt: z.coerce.date(),
+});
+
+export type MediaRegion = z.infer;
+
+/** Fields that are safe to expose to clients. Internal idempotency keys stay private. */
+export const safeMediaRegionSchema = mediaRegionSchema
+ .omit({ detectionKey: true })
+ .extend({ stale: z.boolean() });
+
+export type SafeMediaRegion = z.infer;
+
+export const createManualMediaRegionSchema = z.object({
+ mediaId: z.string().uuid(),
+ bbox: mediaRegionBoundingBoxSchema,
+ label: z.string().trim().min(1).max(200).nullable().optional(),
+ manualReason: z.string().trim().max(500).nullable().optional(),
+});
+
+export type CreateManualMediaRegion = z.infer<
+ typeof createManualMediaRegionSchema
+>;
+
+export const updateMediaRegionSchema = z.object({
+ regionId: z.string().uuid(),
+ expectedRevision: mediaRevisionSchema,
+ bbox: mediaRegionBoundingBoxSchema.optional(),
+ label: z.string().trim().min(1).max(200).nullable().optional(),
+ manualReason: z.string().trim().max(500).nullable().optional(),
+});
+
+export type UpdateMediaRegion = z.infer;
+
+export const deleteMediaRegionSchema = z.object({
+ regionId: z.string().uuid(),
+ expectedRevision: mediaRevisionSchema,
+});
+
+export const mediaRegionRenderProfileSchema = z.object({
+ transparent: z.boolean().default(false),
+});
+
+export type MediaRegionRenderProfile = z.infer<
+ typeof mediaRegionRenderProfileSchema
+>;
+
+export const materializeMediaRegionSchema = z.object({
+ regionId: z.string().uuid(),
+ expectedRevision: mediaRevisionSchema,
+ profile: mediaRegionRenderProfileSchema,
+});
+
+export const materializedMediaRegionSchema = z.object({
+ regionId: z.string().uuid(),
+ mediaId: z.string().uuid(),
+ fileName: z.string(),
+ alreadyExisted: z.boolean(),
+});
+
+export type MaterializedMediaRegion = z.infer<
+ typeof materializedMediaRegionSchema
+>;
+
+export const detectedRegionInputSchema = z.object({
+ bbox: z.object({
+ x1: z.number(),
+ y1: z.number(),
+ x2: z.number(),
+ y2: z.number(),
+ }),
+ label: z.string(),
+ score: z.number().min(0).max(1),
+});
+
+export type DetectedRegionInput = z.infer;
+
+export const mediaRegionRelationSnapshotSchema = z.object({
+ regionId: z.string().uuid(),
+ regionRevision: mediaRevisionSchema,
+ sourceRevision: mediaRevisionSchema,
+ bbox: mediaRegionBoundingBoxSchema,
+ label: z.string().nullable(),
+ profile: mediaRegionRenderProfileSchema,
+ profileVersion: z.string().min(1),
+ rendererVersion: z.string().min(1),
+});
+
+export type MediaRegionRelationSnapshot = z.infer<
+ typeof mediaRegionRelationSnapshotSchema
+>;
diff --git a/packages/core/src/domain/media/revision.ts b/packages/core/src/domain/media/revision.ts
new file mode 100644
index 000000000..35cca4334
--- /dev/null
+++ b/packages/core/src/domain/media/revision.ts
@@ -0,0 +1,127 @@
+const SOURCE_REVISION_VERSION = 1;
+const REGION_REVISION_VERSION = 1;
+const CCIP_INPUT_REVISION_VERSION = 1;
+
+export type MediaSourceRevisionInput = {
+ mediaId: string;
+ mediaSourceId: string;
+ modifiedAt: Date;
+ fileSize: number | null;
+ width: number;
+ height: number;
+};
+
+export type MediaRegionRevisionInput = {
+ sourceRevision: string;
+ kind: "full" | "person" | "manual";
+ x: number | null;
+ y: number | null;
+ width: number | null;
+ height: number | null;
+ label: string | null;
+ detector: string | null;
+ detectorModel: string | null;
+ detectorVersion: string | null;
+ manualReason: string | null;
+};
+
+export type CcipEmbeddingInputRevisionInput = {
+ sourceRevision: string;
+ model: string;
+ embeddingVersion: number;
+ preprocessingProfile: string;
+};
+
+function assertFiniteInteger(value: number, name: string): void {
+ if (!Number.isSafeInteger(value)) {
+ throw new Error(`${name} must be a safe integer`);
+ }
+}
+
+/**
+ * Fixed-order source payload shared by runtime revision checks and SQL
+ * migration fixtures. IDs deliberately prevent a revision from being moved
+ * between otherwise identical media rows.
+ */
+export function canonicalMediaSourceRevisionPayload(
+ input: MediaSourceRevisionInput,
+): string {
+ const modifiedAtMs = input.modifiedAt.getTime();
+ assertFiniteInteger(modifiedAtMs, "modifiedAt");
+ assertFiniteInteger(input.width, "width");
+ assertFiniteInteger(input.height, "height");
+ if (input.fileSize !== null) {
+ assertFiniteInteger(input.fileSize, "fileSize");
+ }
+ return JSON.stringify({
+ version: SOURCE_REVISION_VERSION,
+ mediaId: input.mediaId,
+ mediaSourceId: input.mediaSourceId,
+ modifiedAtMs,
+ fileSize: input.fileSize,
+ width: input.width,
+ height: input.height,
+ });
+}
+
+/** Fixed-order payload for every render-relevant region field. */
+export function canonicalMediaRegionRevisionPayload(
+ input: MediaRegionRevisionInput,
+): string {
+ return JSON.stringify({
+ version: REGION_REVISION_VERSION,
+ sourceRevision: input.sourceRevision,
+ kind: input.kind,
+ x: input.x,
+ y: input.y,
+ width: input.width,
+ height: input.height,
+ label: input.label,
+ detector: input.detector,
+ detectorModel: input.detectorModel,
+ detectorVersion: input.detectorVersion,
+ manualReason: input.manualReason,
+ });
+}
+
+/** Embedding-space identity layered over the underlying media revision. */
+export function canonicalCcipEmbeddingInputRevisionPayload(
+ input: CcipEmbeddingInputRevisionInput,
+): string {
+ assertFiniteInteger(input.embeddingVersion, "embeddingVersion");
+ return JSON.stringify({
+ version: CCIP_INPUT_REVISION_VERSION,
+ sourceRevision: input.sourceRevision,
+ model: input.model,
+ embeddingVersion: input.embeddingVersion,
+ preprocessingProfile: input.preprocessingProfile,
+ });
+}
+
+async function sha256Hex(value: string): Promise {
+ const digest = await globalThis.crypto.subtle.digest(
+ "SHA-256",
+ new TextEncoder().encode(value),
+ );
+ return Array.from(new Uint8Array(digest), (byte) =>
+ byte.toString(16).padStart(2, "0"),
+ ).join("");
+}
+
+export async function createMediaSourceRevision(
+ input: MediaSourceRevisionInput,
+): Promise {
+ return await sha256Hex(canonicalMediaSourceRevisionPayload(input));
+}
+
+export async function createMediaRegionRevision(
+ input: MediaRegionRevisionInput,
+): Promise {
+ return await sha256Hex(canonicalMediaRegionRevisionPayload(input));
+}
+
+export async function createCcipEmbeddingInputRevision(
+ input: CcipEmbeddingInputRevisionInput,
+): Promise {
+ return await sha256Hex(canonicalCcipEmbeddingInputRevisionPayload(input));
+}
diff --git a/packages/core/src/domain/repositories/job-repository.ts b/packages/core/src/domain/repositories/job-repository.ts
index 12f3d8ce6..94d0d342c 100644
--- a/packages/core/src/domain/repositories/job-repository.ts
+++ b/packages/core/src/domain/repositories/job-repository.ts
@@ -1,29 +1,59 @@
-export type JobStatus = "pending" | "in_progress" | "completed" | "failed";
+import type { JobQueueName, JobStatus } from "../jobs/schemas";
+
+export type { JobStatus } from "../jobs/schemas";
export type Job = {
- id: string;
- type: string;
- mediaSourceId: string | null;
- status: JobStatus;
- payload: unknown;
- result: unknown;
- error: string | null;
- createdAt: Date;
- updatedAt: Date;
- parentId: string | null;
+ id: string;
+ type: string;
+ mediaSourceId: string | null;
+ status: JobStatus;
+ payload: unknown;
+ result: unknown;
+ error: string | null;
+ createdAt: Date;
+ updatedAt: Date;
+ parentId: string | null;
+ queueName: JobQueueName | null;
+ targetId: string | null;
+ inputRevision: string | null;
+ dedupeKey: string | null;
+ concurrencyKey: string | null;
+ availableAt: Date;
+ attemptCount: number;
+ maxAttempts: number;
+ leaseDurationMs: number;
+ claimToken: string | null;
+ claimedBy: string | null;
+ claimedAt: Date | null;
+ heartbeatAt: Date | null;
+ errorCode: string | null;
};
export type NewJob = {
- id?: string;
- type: string;
- mediaSourceId?: string | null;
- status?: JobStatus;
- payload?: unknown;
- result?: unknown;
- error?: string | null;
- createdAt?: Date;
- updatedAt?: Date;
- parentId?: string | null;
+ id?: string;
+ type: string;
+ mediaSourceId?: string | null;
+ status?: JobStatus;
+ payload?: unknown;
+ result?: unknown;
+ error?: string | null;
+ createdAt?: Date;
+ updatedAt?: Date;
+ parentId?: string | null;
+ queueName?: JobQueueName | null;
+ targetId?: string | null;
+ inputRevision?: string | null;
+ dedupeKey?: string | null;
+ concurrencyKey?: string | null;
+ availableAt?: Date;
+ attemptCount?: number;
+ maxAttempts?: number;
+ leaseDurationMs?: number;
+ claimToken?: string | null;
+ claimedBy?: string | null;
+ claimedAt?: Date | null;
+ heartbeatAt?: Date | null;
+ errorCode?: string | null;
};
export type BatchProgress = {
@@ -32,39 +62,68 @@ export type BatchProgress = {
total: number;
};
+export type BatchReconciliation = BatchProgress & {
+ status: JobStatus;
+ transitioned: boolean;
+};
+
+export type ClaimFence = {
+ claimToken: string;
+ inputRevision: string | null;
+};
+
+export type ClaimOptions = {
+ excludeTypes?: string[];
+ includeTypes?: string[];
+ excludeLanceDbSourceIds?: string[];
+ queueNames?: JobQueueName[];
+ workerId?: string;
+ now?: Date;
+};
+
+export type ClaimFailure = {
+ error: string;
+ errorCode: string;
+ retryable: boolean;
+ retryAt?: Date;
+};
+
+export type ClaimFailureResult = {
+ status: "pending" | "failed";
+ attemptCount: number;
+};
+
export type IJobRepository = {
create(job: NewJob): Promise;
createIfUnique(job: NewJob): Promise;
- findById(id: string): Promise;
- findPending(
- limit: number,
- options?: {
- excludeTypes?: string[];
- includeTypes?: string[];
- excludeLanceDbSourceIds?: string[];
- },
- ): Promise;
- markAsInProgress(id: string): Promise;
- markAsCompleted(id: string, result?: unknown): Promise;
- markAsFailed(id: string, error: string): Promise;
- update(id: string, data: Partial): Promise;
- incrementProgress(
- id: string,
- progressKey?: string,
- amount?: number,
- ): Promise;
- incrementFailedCount(
- id: string,
- progressKey?: string,
- amount?: number,
- ): Promise;
- claimPending(
- limit: number,
- options?: {
- excludeTypes?: string[];
- includeTypes?: string[];
- excludeLanceDbSourceIds?: string[];
- },
- ): Promise;
- requeueStaleInProgress(olderThan: Date): Promise;
+ createParentWithDispatch(parent: NewJob, dispatch: NewJob): Promise;
+ findById(id: string): Promise;
+ findPending(limit: number, options?: ClaimOptions): Promise;
+ markAsInProgress(id: string): Promise;
+ markAsCompleted(id: string, result?: unknown): Promise;
+ markAsFailed(id: string, error: string): Promise;
+ update(id: string, data: Partial): Promise;
+ heartbeatClaim(id: string, fence: ClaimFence, at?: Date): Promise;
+ completeClaim(id: string, fence: ClaimFence, result?: unknown): Promise;
+ failClaim(
+ id: string,
+ fence: ClaimFence,
+ failure: ClaimFailure,
+ ): Promise;
+ releaseClaim(id: string, fence: ClaimFence, availableAt?: Date): Promise;
+ incrementProgress(
+ id: string,
+ progressKey?: string,
+ amount?: number,
+ ): Promise;
+ incrementFailedCount(
+ id: string,
+ progressKey?: string,
+ amount?: number,
+ ): Promise;
+ recomputeBatchProgress(id: string): Promise;
+ claimPending(limit: number, options?: ClaimOptions): Promise;
+ requeueExpiredLeases(now?: Date): Promise;
+ /** @deprecated Use requeueExpiredLeases; retained for maintenance compatibility. */
+ requeueStaleInProgress(olderThan: Date): Promise;
};
diff --git a/packages/core/src/domain/repositories/media-region-repository.ts b/packages/core/src/domain/repositories/media-region-repository.ts
new file mode 100644
index 000000000..ccc0ed13c
--- /dev/null
+++ b/packages/core/src/domain/repositories/media-region-repository.ts
@@ -0,0 +1,74 @@
+import type { Transaction } from "../interfaces/transaction-manager";
+import type { AddMediaRequest, Media } from "../media/schemas";
+import type {
+ MediaRegion,
+ MediaRegionBoundingBox,
+ MediaRegionRelationSnapshot,
+} from "../media-regions/schemas";
+
+export type NewMediaRegion = {
+ mediaId: string;
+ kind: "person" | "manual";
+ bbox: MediaRegionBoundingBox;
+ sourceWidth: number;
+ sourceHeight: number;
+ sourceModifiedAt: Date;
+ sourceRevision: string;
+ regionRevision: string;
+ label: string | null;
+ manualReason: string | null;
+ detectionKey: string | null;
+ detector: string | null;
+ detectorModel: string | null;
+ detectorVersion: string | null;
+ score: number | null;
+};
+
+export type MediaRegionUpdate = {
+ bbox?: MediaRegionBoundingBox;
+ kind?: "person" | "manual";
+ regionRevision: string;
+ label?: string | null;
+ manualReason?: string | null;
+ detectionKey?: string | null;
+ updatedAt: Date;
+};
+
+export type CreateMaterializedMedia = {
+ media: AddMediaRequest;
+ parentMediaId: string;
+ sourceRegionId: string;
+ derivationKey: string;
+ snapshot: MediaRegionRelationSnapshot;
+};
+
+export interface IMediaRegionRepository {
+ findByMediaId(mediaId: string, tx?: Transaction): Promise;
+ findById(id: string, tx?: Transaction): Promise;
+ create(data: NewMediaRegion, tx?: Transaction): Promise;
+ upsertDetected(data: NewMediaRegion, tx?: Transaction): Promise;
+ deleteDetectedNotIn(
+ mediaId: string,
+ detectionKeys: string[],
+ tx?: Transaction,
+ ): Promise;
+ update(
+ id: string,
+ expectedRevision: string,
+ data: MediaRegionUpdate,
+ tx?: Transaction,
+ ): Promise;
+ delete(
+ id: string,
+ expectedRevision: string,
+ tx?: Transaction,
+ ): Promise