[Test] OpenSQL 지정형 동기와 quorum의 장애 대응 비교

backend/build.gradle

@@ -65,7 +65,7 @@ dependencies {
 
 tasks.named('test') {
 	useJUnitPlatform {
-		excludeTags 'permission-standby', 'permission-durability', 'benchmark', 'dashboard-cloud-load', 'minio-integration', 'claim-concurrency', 'local-e2e', 'storage-worker-e2e', 'real-pdf-version-e2e', 'vector-search-performance', 'vector-storage-performance', 'worker-indexing-throughput', 'worker-horizontal-scaling', 'worker-queue-backpressure', 'document-indexing-e2e-load', 'chunk-quality-performance', 'section-path-quality', 'opensql-ha-connection', 'opensql-ha-contract', 'opensql-outbox-isolated'
+		excludeTags 'permission-ha', 'permission-standby', 'permission-durability', 'benchmark', 'dashboard-cloud-load', 'minio-integration', 'claim-concurrency', 'local-e2e', 'storage-worker-e2e', 'real-pdf-version-e2e', 'vector-search-performance', 'vector-storage-performance', 'worker-indexing-throughput', 'worker-horizontal-scaling', 'worker-queue-backpressure', 'document-indexing-e2e-load', 'chunk-quality-performance', 'section-path-quality', 'opensql-ha-connection', 'opensql-ha-contract', 'opensql-outbox-isolated'
 	}
 }
 
@@ -630,3 +630,34 @@ tasks.register('permissionStandbyTest', Test) {
     outputs.upToDateWhen { false }
     testLogging { events 'passed', 'failed'; showStandardStreams = true }
 }
+
+// 실제 라이선스 호스트의 격리 OpenSQL·OpenProxy에만 연결하는 수동 HA 정책 비교.
+tasks.register('permissionHaTest', Test) {
+    group = 'verification'
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+    useJUnitPlatform { includeTags 'permission-ha' }
+    maxParallelForks = 1
+    outputs.upToDateWhen { false }
+    testLogging { events 'passed', 'failed'; showStandardStreams = true }
+    doFirst {
+        if (!(System.getenv('PERMISSION_HA_JDBC_URL') ?: '').matches('jdbc:postgresql://127\\.0\\.0\\.1:1844[23]/permission_test.*')) {
+            throw new GradleException('전용 루프백 격리 OpenProxy URL이 필요합니다.')
+        }
+        // Flyway와 fixture 준비도 원본 DB에 닿지 않도록 전용 루프백 경로만 허용한다.
+        if (!(System.getenv('PERMISSION_HA_MIGRATION_JDBC_URL') ?: '').matches('jdbc:postgresql://127\\.0\\.0\\.1:1843[123]/postgres')) {
+            throw new GradleException('전용 루프백 격리 migration URL이 필요합니다.')
+        }
+        if (!(System.getenv('PERMISSION_HA_MODE') in ['designated', 'quorum']) ||
+            !(System.getenv('PERMISSION_HA_SCENARIO') in ['standby-loss', 'primary-loss'])) {
+            throw new GradleException('지원하는 격리 HA 모드와 장애 시나리오가 필요합니다.')
+        }
+        def runDirectory = file(System.getenv('PERMISSION_HA_RUN_DIR') ?: '.')
+        if (['requests.jsonl', 'summary.txt', 'ready', 'fault.done'].any { new File(runDirectory, it).exists() }) {
+            throw new GradleException('새 회차 디렉터리가 필요합니다. 이전 원장은 덮어쓸 수 없습니다.')
+        }
+        ['PERMISSION_HA_PASSWORD','PERMISSION_HA_RUN_DIR','PERMISSION_HA_MODE','PERMISSION_HA_SCENARIO'].each { name ->
+            if (!System.getenv(name)?.trim()) throw new GradleException('격리 HA 실행 설정이 필요합니다: ' + name)
+        }
+    }
+}

backend/src/test/java/com/opensource/docgrid/domain/permission/integration/OpenSqlSynchronousPolicyComparisonTest.java

@@ -0,0 +1,201 @@
+package com.opensource.docgrid.domain.permission.integration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import java.nio.file.*;
+import java.time.OffsetDateTime;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import javax.sql.DataSource;
+import org.junit.jupiter.api.*;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+import com.opensource.docgrid.domain.auth.resolver.CurrentUserArgumentResolver;
+import com.opensource.docgrid.domain.document.entity.Document;
+import com.opensource.docgrid.domain.document.enums.*;
+import com.opensource.docgrid.domain.document.repository.DocumentRepository;
+import com.opensource.docgrid.domain.permission.controller.PermissionController;
+import com.opensource.docgrid.domain.permission.dto.request.GrantPermissionRequest;
+import com.opensource.docgrid.domain.permission.enums.*;
+import com.opensource.docgrid.domain.permission.service.DocumentPermissionRevocationFacade;
+import com.opensource.docgrid.domain.permission.service.command.*;
+import com.opensource.docgrid.domain.permission.service.query.*;
+import com.opensource.docgrid.domain.user.entity.User;
+import com.opensource.docgrid.domain.user.enums.UserStatus;
+import com.opensource.docgrid.domain.user.repository.UserRepository;
+import com.opensource.docgrid.global.exception.GlobalExceptionHandler;
+import com.zaxxer.hikari.HikariDataSource;
+
+/**
+ * 실제 격리 OpenSQL·OpenProxy에서 같은 MVC·JPA 회수 부하의 성공 보존·재확정·인가를 계측한다.
+ * 장애는 외부 제어기가 시험 컨테이너에만 주입하고 요청 원장은 DB 밖에 즉시 기록한다.
+ */
+@Tag("integration")
+@Tag("permission-ha")
+@SpringBootTest(classes = PermissionDurabilityIntegrationTest.Config.class, properties = {
+    "spring.datasource.url=${PERMISSION_HA_JDBC_URL}", "spring.datasource.username=postgres",
+    "spring.datasource.password=${PERMISSION_HA_PASSWORD}", "spring.datasource.hikari.maximum-pool-size=5",
+    "spring.datasource.hikari.connection-timeout=5000", "spring.datasource.hikari.validation-timeout=2000",
+    "spring.jpa.hibernate.ddl-auto=validate", "spring.jpa.open-in-view=false",
+    "spring.flyway.locations=classpath:db/migration", "spring.flyway.enabled=true",
+    "spring.flyway.url=${PERMISSION_HA_MIGRATION_JDBC_URL}", "spring.flyway.user=postgres",
+    "spring.flyway.password=${PERMISSION_HA_PASSWORD}",
+    "permission.revocation.synchronous-enabled=true", "management.server.port=0", "logging.level.root=OFF"
+})
+@Timeout(360)
+class OpenSqlSynchronousPolicyComparisonTest {
+    private static final int SAMPLE_COUNT = 48;
+    @Autowired DocumentPermissionCommandService commands;
+    @Autowired DocumentPermissionRevocationFacade facade;
+    @Autowired PermissionQueryService queries;
+    @Autowired PrimaryDocumentPermissionQueryService primary;
+    @Autowired UserRepository users;
+    @Autowired DocumentRepository documents;
+    @Autowired PlatformTransactionManager manager;
+    @Autowired DataSource dataSource;
+    private final AtomicInteger completed = new AtomicInteger();
+    private final AtomicInteger forbidden = new AtomicInteger();
+    private final AtomicInteger authorizationUnavailable = new AtomicInteger();
+    private final List<Result> results = Collections.synchronizedList(new ArrayList<>());
+    private MockMvc mvc;
+    private Path directory;
+
+    @Test
+    @DisplayName("동일 회수 부하 중 장애가 있어도 확정 요청이 보존되고 불명 요청은 같은 키로 수렴한다")
+    void compareAcknowledgedRevocations() throws Exception {
+        directory = Path.of(System.getenv("PERMISSION_HA_RUN_DIR"));
+        Files.createDirectories(directory);
+        var controller = new PermissionController(mock(CollectionPermissionCommandService.class), commands,
+            queries, mock(PermissionTargetUserQueryService.class), facade);
+        mvc = MockMvcBuilders.standaloneSetup(controller).setCustomArgumentResolvers(new CurrentUserArgumentResolver())
+            .setControllerAdvice(new GlobalExceptionHandler()).build();
+        event("fixture-start", "samples=" + SAMPLE_COUNT);
+        List<Fixture> fixtures = seed();
+        event("fixture-ready", "samples=" + fixtures.size());
+        HikariDataSource pool = dataSource.unwrap(HikariDataSource.class);
+        AtomicInteger maxActive = new AtomicInteger();
+        AtomicInteger maxPending = new AtomicInteger();
+        var monitor = Executors.newSingleThreadScheduledExecutor();
+        monitor.scheduleAtFixedRate(() -> {
+            maxActive.accumulateAndGet(pool.getHikariPoolMXBean().getActiveConnections(), Math::max);
+            maxPending.accumulateAndGet(pool.getHikariPoolMXBean().getThreadsAwaitingConnection(), Math::max);
+        }, 0, 100, TimeUnit.MILLISECONDS);
+        var workers = Executors.newFixedThreadPool(2);
+        try {
+            // 1. 같은 Binary·표본·동시성으로 실행하며 매 응답을 DB 밖에 즉시 보존한다.
+            var futures = new ArrayList<Future<?>>();
+            for (Fixture f : fixtures) futures.add(workers.submit(() -> execute(f, false)));
+            for (Future<?> future : futures) future.get(150, TimeUnit.SECONDS);
+            // 2. 외부 장애 제어기의 복구 관측이 끝나기 전 결과를 확정하지 않는다.
+            long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(90);
+            while (!Files.exists(directory.resolve("fault.done")) && System.nanoTime() < deadline) Thread.sleep(100);
+            assertThat(Files.exists(directory.resolve("fault.done"))).isTrue();
+            // 3. 결과 불명은 원장을 읽기만 하지 않고 같은 키 WAL 재주장으로 해소한다.
+            List<Result> initial = List.copyOf(results);
+            for (Result result : initial) {
+                if (result.status()!=204) {
+                    for (int retry=0;retry<4;retry++) {
+                        if (execute(result.fixture(), true).status()==204) break;
+                        Thread.sleep(500);
+                    }
+                }
+            }
+            int acknowledgedPreserved=new TransactionTemplate(manager).execute(status -> {
+                // 원장 대조도 read-write Transaction으로 현재 Primary 연결에 고정한다.
+                JdbcTemplate jdbc=new JdbcTemplate(dataSource);
+                assertThat(jdbc.queryForObject("SELECT pg_is_in_recovery()",Boolean.class)).isFalse();
+                String marks=String.join(",",Collections.nCopies(fixtures.size(),"?"));
+                Object[] keys=fixtures.stream().map(Fixture::key).toArray();
+                Object[] ids=fixtures.stream().map(Fixture::permission).toArray();
+                long receipt=jdbc.queryForObject("SELECT count(*) FROM permission_revocation_requests WHERE request_id IN ("+marks+")",Long.class,keys);
+                long remaining=jdbc.queryForObject("SELECT count(*) FROM document_permissions WHERE id IN ("+marks+")",Long.class,ids);
+                assertThat(receipt).isEqualTo(SAMPLE_COUNT);assertThat(remaining).isZero();
+                return (int)receipt;
+            });
+            assertThat(forbidden.get()).isZero();
+            long accepted = initial.stream().filter(r -> r.status()==204).count();
+            long unknown = initial.size()-accepted;
+            double[] latency = initial.stream().mapToDouble(Result::latencyMs).sorted().toArray();
+            String summary="initial_requests="+initial.size()+" initial_204="+accepted+" initial_unknown="+unknown
+                +" final_receipts="+acknowledgedPreserved+" final_permissions=0 forbidden_content="+forbidden.get()
+                +" auth_unavailable="+authorizationUnavailable.get()+" p50_ms="+percentile(latency,.50)+" p95_ms="+percentile(latency,.95)
+                +" hikari_max_active="+maxActive.get()+" hikari_max_pending="+maxPending.get();
+            Files.writeString(directory.resolve("summary.txt"), summary+"\n", StandardOpenOption.CREATE_NEW);
+            event("summary",summary);
+        } finally {
+            workers.shutdownNow();monitor.shutdownNow();
+        }
+    }
+
+    private List<Fixture> seed() throws Exception {
+        // 합성 fixture 준비는 직접 Primary에서 한 번의 배치로 수행하며 측정 부하에 포함하지 않는다.
+        try (var connection=java.sql.DriverManager.getConnection(System.getenv("PERMISSION_HA_MIGRATION_JDBC_URL"),
+                "postgres",System.getenv("PERMISSION_HA_PASSWORD"))) {
+            connection.setAutoCommit(false);
+            String token=UUID.randomUUID().toString();
+            Long owner;Long target;
+            try(var statement=connection.prepareStatement("INSERT INTO users(email,password_hash,name,status,created_at,updated_at) VALUES(?, '!disabled!', ?, 'ACTIVE', CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) RETURNING id")) {
+                statement.setString(1,"owner-"+token+"@example.invalid");statement.setString(2,"합성 소유자");
+                try(var rows=statement.executeQuery()){rows.next();owner=rows.getLong(1);}
+                statement.setString(1,"target-"+token+"@example.invalid");statement.setString(2,"합성 대상자");
+                try(var rows=statement.executeQuery()){rows.next();target=rows.getLong(1);}
+            }
+            List<Fixture> fixtures=new ArrayList<>();
+            try(var statement=connection.prepareStatement("WITH docs AS (INSERT INTO documents(owner_user_id,title,document_type,source_type,status,visibility,created_at,updated_at) SELECT ?, '격리 합성 권한 문서', 'TXT','UPLOAD','INDEXED','PRIVATE',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP FROM generate_series(1,?) RETURNING id) INSERT INTO document_permissions(document_id,target_type,user_id,permission_type,can_read,can_write,can_admin,granted_by,granted_at,created_at,updated_at) SELECT id,'USER',?,'READ',true,false,false,?,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP FROM docs RETURNING id,document_id")) {
+                statement.setLong(1,owner);statement.setInt(2,SAMPLE_COUNT);statement.setLong(3,target);statement.setLong(4,owner);
+                try(var rows=statement.executeQuery()){while(rows.next())fixtures.add(new Fixture(fixtures.size(),owner,target,rows.getLong(2),rows.getLong(1),UUID.randomUUID()));}
+            }
+            try(var statement=connection.prepareStatement("INSERT INTO user_document_access_cache(user_id,document_id,can_read,can_write,can_admin,source_type,source_id,computed_at,created_at,updated_at) SELECT user_id,document_id,true,false,false,'DIRECT_DOCUMENT_PERMISSION',id,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP FROM document_permissions WHERE user_id=?")) {
+                statement.setLong(1,target);statement.executeUpdate();
+            }
+            connection.commit();return fixtures;
+        }
+    }
+
+    private Result execute(Fixture f, boolean retry) {
+        var authentication=new UsernamePasswordAuthenticationToken("synthetic", "", List.of());
+        authentication.setDetails(f.owner());SecurityContextHolder.getContext().setAuthentication(authentication);
+        long start=System.nanoTime();int code;
+        try {
+            code=mvc.perform(delete("/permissions/documents/{d}/{p}",f.document(),f.permission())
+                .header("Idempotency-Key",f.key().toString()).accept("application/json")).andReturn().getResponse().getStatus();
+        } catch(Exception exception) {code=599;}
+        finally {SecurityContextHolder.clearContext();}
+        double elapsed=(System.nanoTime()-start)/1_000_000.0;
+        Result result=new Result(f,code,elapsed);
+        results.add(result);
+        // 외부 원장은 합성 요청 키·논리 표본 번호·상태·시간만 기록한다.
+        synchronized(this) {
+            try { Files.writeString(directory.resolve("requests.jsonl"),
+                "{\"at\":\""+OffsetDateTime.now()+"\",\"sample\":"+f.ordinal()+",\"key\":\""+f.key()+"\",\"retry\":"+retry+",\"status\":"+code+",\"latency_ms\":"+elapsed+"}\n",
+                StandardOpenOption.CREATE,StandardOpenOption.APPEND);
+                if(!retry && completed.incrementAndGet()==4) Files.writeString(directory.resolve("ready"),"first_four_requests_completed\n",StandardOpenOption.CREATE_NEW);
+            } catch(Exception exception) {throw new IllegalStateException("External ledger write failed",exception);}
+        }
+        if(code==204) {
+            try {if(primary.canReadDocument(f.target(),f.document()))forbidden.incrementAndGet();}
+            catch(Exception exception) {authorizationUnavailable.incrementAndGet();}
+        }
+        return result;
+    }
+    private static double percentile(double[] values,double quantile) {
+        return values[Math.max(0,(int)Math.ceil(values.length*quantile)-1)];
+    }
+    private static void event(String phase,String detail) {
+        System.out.println("EVIDENCE "+OffsetDateTime.now()+" mode="+System.getenv("PERMISSION_HA_MODE")
+            +" scenario="+System.getenv("PERMISSION_HA_SCENARIO")+" phase="+phase+" "+detail);
+    }
+    /** 합성 실행에 필요한 ID만 보관하며 실제 계정·인프라 주소는 포함하지 않는다. */
+    private record Fixture(int ordinal,Long owner,Long target,Long document,Long permission,UUID key) {}
+    /** 한 번의 실제 MVC 응답과 지연을 외부 원장·최종 DB 대조에 연결한다. */
+    private record Result(Fixture fixture,int status,double latencyMs) {}
+}