diff --git a/backend/build.gradle b/backend/build.gradle index 345d2cbe..70a77ad9 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -65,7 +65,7 @@ dependencies { tasks.named('test') { useJUnitPlatform { - excludeTags 'permission-standby', 'permission-durability', 'benchmark', 'dashboard-cloud-load', 'minio-integration', 'claim-concurrency', 'local-e2e', 'storage-worker-e2e', 'real-pdf-version-e2e', 'vector-search-performance', 'vector-storage-performance', 'worker-indexing-throughput', 'worker-horizontal-scaling', 'worker-queue-backpressure', 'document-indexing-e2e-load', 'chunk-quality-performance', 'section-path-quality', 'opensql-ha-connection', 'opensql-ha-contract', 'opensql-outbox-isolated' + excludeTags 'permission-ha', 'permission-standby', 'permission-durability', 'benchmark', 'dashboard-cloud-load', 'minio-integration', 'claim-concurrency', 'local-e2e', 'storage-worker-e2e', 'real-pdf-version-e2e', 'vector-search-performance', 'vector-storage-performance', 'worker-indexing-throughput', 'worker-horizontal-scaling', 'worker-queue-backpressure', 'document-indexing-e2e-load', 'chunk-quality-performance', 'section-path-quality', 'opensql-ha-connection', 'opensql-ha-contract', 'opensql-outbox-isolated' } } @@ -630,3 +630,34 @@ tasks.register('permissionStandbyTest', Test) { outputs.upToDateWhen { false } testLogging { events 'passed', 'failed'; showStandardStreams = true } } + +// 실제 라이선스 호스트의 격리 OpenSQL·OpenProxy에만 연결하는 수동 HA 정책 비교. +tasks.register('permissionHaTest', Test) { + group = 'verification' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + useJUnitPlatform { includeTags 'permission-ha' } + maxParallelForks = 1 + outputs.upToDateWhen { false } + testLogging { events 'passed', 'failed'; showStandardStreams = true } + doFirst { + if (!(System.getenv('PERMISSION_HA_JDBC_URL') ?: '').matches('jdbc:postgresql://127\\.0\\.0\\.1:1844[23]/permission_test.*')) { + throw new GradleException('전용 루프백 격리 OpenProxy URL이 필요합니다.') + } + // Flyway와 fixture 준비도 원본 DB에 닿지 않도록 전용 루프백 경로만 허용한다. + if (!(System.getenv('PERMISSION_HA_MIGRATION_JDBC_URL') ?: '').matches('jdbc:postgresql://127\\.0\\.0\\.1:1843[123]/postgres')) { + throw new GradleException('전용 루프백 격리 migration URL이 필요합니다.') + } + if (!(System.getenv('PERMISSION_HA_MODE') in ['designated', 'quorum']) || + !(System.getenv('PERMISSION_HA_SCENARIO') in ['standby-loss', 'primary-loss'])) { + throw new GradleException('지원하는 격리 HA 모드와 장애 시나리오가 필요합니다.') + } + def runDirectory = file(System.getenv('PERMISSION_HA_RUN_DIR') ?: '.') + if (['requests.jsonl', 'summary.txt', 'ready', 'fault.done'].any { new File(runDirectory, it).exists() }) { + throw new GradleException('새 회차 디렉터리가 필요합니다. 이전 원장은 덮어쓸 수 없습니다.') + } + ['PERMISSION_HA_PASSWORD','PERMISSION_HA_RUN_DIR','PERMISSION_HA_MODE','PERMISSION_HA_SCENARIO'].each { name -> + if (!System.getenv(name)?.trim()) throw new GradleException('격리 HA 실행 설정이 필요합니다: ' + name) + } + } +} diff --git a/backend/src/test/java/com/opensource/docgrid/domain/permission/integration/OpenSqlSynchronousPolicyComparisonTest.java b/backend/src/test/java/com/opensource/docgrid/domain/permission/integration/OpenSqlSynchronousPolicyComparisonTest.java new file mode 100644 index 00000000..b2dafe64 --- /dev/null +++ b/backend/src/test/java/com/opensource/docgrid/domain/permission/integration/OpenSqlSynchronousPolicyComparisonTest.java @@ -0,0 +1,201 @@ +package com.opensource.docgrid.domain.permission.integration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import java.nio.file.*; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import javax.sql.DataSource; +import org.junit.jupiter.api.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import com.opensource.docgrid.domain.auth.resolver.CurrentUserArgumentResolver; +import com.opensource.docgrid.domain.document.entity.Document; +import com.opensource.docgrid.domain.document.enums.*; +import com.opensource.docgrid.domain.document.repository.DocumentRepository; +import com.opensource.docgrid.domain.permission.controller.PermissionController; +import com.opensource.docgrid.domain.permission.dto.request.GrantPermissionRequest; +import com.opensource.docgrid.domain.permission.enums.*; +import com.opensource.docgrid.domain.permission.service.DocumentPermissionRevocationFacade; +import com.opensource.docgrid.domain.permission.service.command.*; +import com.opensource.docgrid.domain.permission.service.query.*; +import com.opensource.docgrid.domain.user.entity.User; +import com.opensource.docgrid.domain.user.enums.UserStatus; +import com.opensource.docgrid.domain.user.repository.UserRepository; +import com.opensource.docgrid.global.exception.GlobalExceptionHandler; +import com.zaxxer.hikari.HikariDataSource; + +/** + * 실제 격리 OpenSQL·OpenProxy에서 같은 MVC·JPA 회수 부하의 성공 보존·재확정·인가를 계측한다. + * 장애는 외부 제어기가 시험 컨테이너에만 주입하고 요청 원장은 DB 밖에 즉시 기록한다. + */ +@Tag("integration") +@Tag("permission-ha") +@SpringBootTest(classes = PermissionDurabilityIntegrationTest.Config.class, properties = { + "spring.datasource.url=${PERMISSION_HA_JDBC_URL}", "spring.datasource.username=postgres", + "spring.datasource.password=${PERMISSION_HA_PASSWORD}", "spring.datasource.hikari.maximum-pool-size=5", + "spring.datasource.hikari.connection-timeout=5000", "spring.datasource.hikari.validation-timeout=2000", + "spring.jpa.hibernate.ddl-auto=validate", "spring.jpa.open-in-view=false", + "spring.flyway.locations=classpath:db/migration", "spring.flyway.enabled=true", + "spring.flyway.url=${PERMISSION_HA_MIGRATION_JDBC_URL}", "spring.flyway.user=postgres", + "spring.flyway.password=${PERMISSION_HA_PASSWORD}", + "permission.revocation.synchronous-enabled=true", "management.server.port=0", "logging.level.root=OFF" +}) +@Timeout(360) +class OpenSqlSynchronousPolicyComparisonTest { + private static final int SAMPLE_COUNT = 48; + @Autowired DocumentPermissionCommandService commands; + @Autowired DocumentPermissionRevocationFacade facade; + @Autowired PermissionQueryService queries; + @Autowired PrimaryDocumentPermissionQueryService primary; + @Autowired UserRepository users; + @Autowired DocumentRepository documents; + @Autowired PlatformTransactionManager manager; + @Autowired DataSource dataSource; + private final AtomicInteger completed = new AtomicInteger(); + private final AtomicInteger forbidden = new AtomicInteger(); + private final AtomicInteger authorizationUnavailable = new AtomicInteger(); + private final List results = Collections.synchronizedList(new ArrayList<>()); + private MockMvc mvc; + private Path directory; + + @Test + @DisplayName("동일 회수 부하 중 장애가 있어도 확정 요청이 보존되고 불명 요청은 같은 키로 수렴한다") + void compareAcknowledgedRevocations() throws Exception { + directory = Path.of(System.getenv("PERMISSION_HA_RUN_DIR")); + Files.createDirectories(directory); + var controller = new PermissionController(mock(CollectionPermissionCommandService.class), commands, + queries, mock(PermissionTargetUserQueryService.class), facade); + mvc = MockMvcBuilders.standaloneSetup(controller).setCustomArgumentResolvers(new CurrentUserArgumentResolver()) + .setControllerAdvice(new GlobalExceptionHandler()).build(); + event("fixture-start", "samples=" + SAMPLE_COUNT); + List fixtures = seed(); + event("fixture-ready", "samples=" + fixtures.size()); + HikariDataSource pool = dataSource.unwrap(HikariDataSource.class); + AtomicInteger maxActive = new AtomicInteger(); + AtomicInteger maxPending = new AtomicInteger(); + var monitor = Executors.newSingleThreadScheduledExecutor(); + monitor.scheduleAtFixedRate(() -> { + maxActive.accumulateAndGet(pool.getHikariPoolMXBean().getActiveConnections(), Math::max); + maxPending.accumulateAndGet(pool.getHikariPoolMXBean().getThreadsAwaitingConnection(), Math::max); + }, 0, 100, TimeUnit.MILLISECONDS); + var workers = Executors.newFixedThreadPool(2); + try { + // 1. 같은 Binary·표본·동시성으로 실행하며 매 응답을 DB 밖에 즉시 보존한다. + var futures = new ArrayList>(); + for (Fixture f : fixtures) futures.add(workers.submit(() -> execute(f, false))); + for (Future future : futures) future.get(150, TimeUnit.SECONDS); + // 2. 외부 장애 제어기의 복구 관측이 끝나기 전 결과를 확정하지 않는다. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(90); + while (!Files.exists(directory.resolve("fault.done")) && System.nanoTime() < deadline) Thread.sleep(100); + assertThat(Files.exists(directory.resolve("fault.done"))).isTrue(); + // 3. 결과 불명은 원장을 읽기만 하지 않고 같은 키 WAL 재주장으로 해소한다. + List initial = List.copyOf(results); + for (Result result : initial) { + if (result.status()!=204) { + for (int retry=0;retry<4;retry++) { + if (execute(result.fixture(), true).status()==204) break; + Thread.sleep(500); + } + } + } + int acknowledgedPreserved=new TransactionTemplate(manager).execute(status -> { + // 원장 대조도 read-write Transaction으로 현재 Primary 연결에 고정한다. + JdbcTemplate jdbc=new JdbcTemplate(dataSource); + assertThat(jdbc.queryForObject("SELECT pg_is_in_recovery()",Boolean.class)).isFalse(); + String marks=String.join(",",Collections.nCopies(fixtures.size(),"?")); + Object[] keys=fixtures.stream().map(Fixture::key).toArray(); + Object[] ids=fixtures.stream().map(Fixture::permission).toArray(); + long receipt=jdbc.queryForObject("SELECT count(*) FROM permission_revocation_requests WHERE request_id IN ("+marks+")",Long.class,keys); + long remaining=jdbc.queryForObject("SELECT count(*) FROM document_permissions WHERE id IN ("+marks+")",Long.class,ids); + assertThat(receipt).isEqualTo(SAMPLE_COUNT);assertThat(remaining).isZero(); + return (int)receipt; + }); + assertThat(forbidden.get()).isZero(); + long accepted = initial.stream().filter(r -> r.status()==204).count(); + long unknown = initial.size()-accepted; + double[] latency = initial.stream().mapToDouble(Result::latencyMs).sorted().toArray(); + String summary="initial_requests="+initial.size()+" initial_204="+accepted+" initial_unknown="+unknown + +" final_receipts="+acknowledgedPreserved+" final_permissions=0 forbidden_content="+forbidden.get() + +" auth_unavailable="+authorizationUnavailable.get()+" p50_ms="+percentile(latency,.50)+" p95_ms="+percentile(latency,.95) + +" hikari_max_active="+maxActive.get()+" hikari_max_pending="+maxPending.get(); + Files.writeString(directory.resolve("summary.txt"), summary+"\n", StandardOpenOption.CREATE_NEW); + event("summary",summary); + } finally { + workers.shutdownNow();monitor.shutdownNow(); + } + } + + private List seed() throws Exception { + // 합성 fixture 준비는 직접 Primary에서 한 번의 배치로 수행하며 측정 부하에 포함하지 않는다. + try (var connection=java.sql.DriverManager.getConnection(System.getenv("PERMISSION_HA_MIGRATION_JDBC_URL"), + "postgres",System.getenv("PERMISSION_HA_PASSWORD"))) { + connection.setAutoCommit(false); + String token=UUID.randomUUID().toString(); + Long owner;Long target; + try(var statement=connection.prepareStatement("INSERT INTO users(email,password_hash,name,status,created_at,updated_at) VALUES(?, '!disabled!', ?, 'ACTIVE', CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) RETURNING id")) { + statement.setString(1,"owner-"+token+"@example.invalid");statement.setString(2,"합성 소유자"); + try(var rows=statement.executeQuery()){rows.next();owner=rows.getLong(1);} + statement.setString(1,"target-"+token+"@example.invalid");statement.setString(2,"합성 대상자"); + try(var rows=statement.executeQuery()){rows.next();target=rows.getLong(1);} + } + List fixtures=new ArrayList<>(); + try(var statement=connection.prepareStatement("WITH docs AS (INSERT INTO documents(owner_user_id,title,document_type,source_type,status,visibility,created_at,updated_at) SELECT ?, '격리 합성 권한 문서', 'TXT','UPLOAD','INDEXED','PRIVATE',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP FROM generate_series(1,?) RETURNING id) INSERT INTO document_permissions(document_id,target_type,user_id,permission_type,can_read,can_write,can_admin,granted_by,granted_at,created_at,updated_at) SELECT id,'USER',?,'READ',true,false,false,?,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP FROM docs RETURNING id,document_id")) { + statement.setLong(1,owner);statement.setInt(2,SAMPLE_COUNT);statement.setLong(3,target);statement.setLong(4,owner); + try(var rows=statement.executeQuery()){while(rows.next())fixtures.add(new Fixture(fixtures.size(),owner,target,rows.getLong(2),rows.getLong(1),UUID.randomUUID()));} + } + try(var statement=connection.prepareStatement("INSERT INTO user_document_access_cache(user_id,document_id,can_read,can_write,can_admin,source_type,source_id,computed_at,created_at,updated_at) SELECT user_id,document_id,true,false,false,'DIRECT_DOCUMENT_PERMISSION',id,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP FROM document_permissions WHERE user_id=?")) { + statement.setLong(1,target);statement.executeUpdate(); + } + connection.commit();return fixtures; + } + } + + private Result execute(Fixture f, boolean retry) { + var authentication=new UsernamePasswordAuthenticationToken("synthetic", "", List.of()); + authentication.setDetails(f.owner());SecurityContextHolder.getContext().setAuthentication(authentication); + long start=System.nanoTime();int code; + try { + code=mvc.perform(delete("/permissions/documents/{d}/{p}",f.document(),f.permission()) + .header("Idempotency-Key",f.key().toString()).accept("application/json")).andReturn().getResponse().getStatus(); + } catch(Exception exception) {code=599;} + finally {SecurityContextHolder.clearContext();} + double elapsed=(System.nanoTime()-start)/1_000_000.0; + Result result=new Result(f,code,elapsed); + results.add(result); + // 외부 원장은 합성 요청 키·논리 표본 번호·상태·시간만 기록한다. + synchronized(this) { + try { Files.writeString(directory.resolve("requests.jsonl"), + "{\"at\":\""+OffsetDateTime.now()+"\",\"sample\":"+f.ordinal()+",\"key\":\""+f.key()+"\",\"retry\":"+retry+",\"status\":"+code+",\"latency_ms\":"+elapsed+"}\n", + StandardOpenOption.CREATE,StandardOpenOption.APPEND); + if(!retry && completed.incrementAndGet()==4) Files.writeString(directory.resolve("ready"),"first_four_requests_completed\n",StandardOpenOption.CREATE_NEW); + } catch(Exception exception) {throw new IllegalStateException("External ledger write failed",exception);} + } + if(code==204) { + try {if(primary.canReadDocument(f.target(),f.document()))forbidden.incrementAndGet();} + catch(Exception exception) {authorizationUnavailable.incrementAndGet();} + } + return result; + } + private static double percentile(double[] values,double quantile) { + return values[Math.max(0,(int)Math.ceil(values.length*quantile)-1)]; + } + private static void event(String phase,String detail) { + System.out.println("EVIDENCE "+OffsetDateTime.now()+" mode="+System.getenv("PERMISSION_HA_MODE") + +" scenario="+System.getenv("PERMISSION_HA_SCENARIO")+" phase="+phase+" "+detail); + } + /** 합성 실행에 필요한 ID만 보관하며 실제 계정·인프라 주소는 포함하지 않는다. */ + private record Fixture(int ordinal,Long owner,Long target,Long document,Long permission,UUID key) {} + /** 한 번의 실제 MVC 응답과 지연을 외부 원장·최종 DB 대조에 연결한다. */ + private record Result(Fixture fixture,int status,double latencyMs) {} +} diff --git a/docs/test-results/opensql-sync-quorum-20261017/README.md b/docs/test-results/opensql-sync-quorum-20261017/README.md new file mode 100644 index 00000000..73492370 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/README.md @@ -0,0 +1,97 @@ +# OpenSQL 지정형 동기·quorum 장애 비교 + +**역할 호환 어댑터를 경유한 실제 격리 OpenSQL·OpenProxy에서 4회차가 통과했습니다.** 최초 요청 192개 중 174개는 204였고, Primary 상실 때의 503 18개는 같은 키 재시도로 모두 확정했습니다. 최초 204 키 174개가 모두 보존됐으며 최종 요청 키는 192개, 회수 대상 권한 행과 최종 인가 허용은 0개입니다. + +> **원본 OpenProxy 그대로는 두 동기 모드 모두 시작에 실패합니다.** `sync_standby`와 `quorum_standby` parser 오류를 실제 관측했으며 [호환 어댑터와 원본 실패 기록](../patroni-role-compatibility-20261017/README.md)에 분리했습니다. 아래 표는 어댑터 경유 결과이며 공급사 공식 지원이나 공개 배포 완료를 뜻하지 않습니다. + +## 실제 결과 + +| 회차 | 최초 204 | 최초 503 | 같은 키 재시도 | 최종 키 대조 | 최초 응답 p95 | 종료 확인→쓰기 재개 | Hikari active/pending 최대 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| [지정형 · Standby 1대 중단](designated-standby-run05/control.jsonl) | 48/48 | 0 | 0 | 48/48 | 2906.4 ms | 3.527초 | 2/0 | +| [quorum · Standby 1대 중단](quorum-standby-run01/control.jsonl) | 48/48 | 0 | 0 | 48/48 | 2920.4 ms | 3.474초 | 2/0 | +| [지정형 · Primary 상실](designated-primary-run01/control.jsonl) | 39/48 | 9 | 9 | 48/48 | 5018.8 ms | 26.067초 | 2/2 | +| [quorum · Primary 상실](quorum-primary-run01/control.jsonl) | 39/48 | 9 | 9 | 48/48 | 5019.7 ms | 25.902초 | 2/2 | + +[집계 원장](aggregate.json), [판정·시간 정의](measurement-protocol.md). + +- 각 회차는 합성 권한 48개·동시성 2이며 동일 애플리케이션/시험 코드 `bb3c73521155a64e9675a28a32c9f673f9a47e0f`를 사용했습니다. +- 네 회차 모두 strict=true, synchronous_node_count=1, on. 최초 Primary·호스트·스키마·데이터량을 같게 초기화했습니다. 지정형은 실제 sync 후보 1대, quorum은 실제 ANY 1 후보 2대를 SQL로 확인했습니다. +- Primary 자동 승격 관측은 지정형 **20.385초**, quorum **18.695초**입니다. 쓰기 재개는 각각 **26.067초**, **25.902초**입니다. 표본 1회씩이며 이 차이로 성능 우위를 일반화하지 않습니다. +- 장애 중 204 뒤 별도 인가 확인이 불가능한 경우는 Primary 회차마다 **2건**이었습니다. 허용으로 추정하지 않았으며 별도로 집계했습니다. `forbidden_content` 원시 카운터는 **최종 인가가 true인 횟수**이고, GCP 네 조회 API의 실제 본문 반환량을 계측한 값이 아닙니다. +- 503은 롤백 확정으로 표현하지 않았습니다. 같은 요청 키의 WAL 재주장과 독립 JDBC 키 감사로 최종 상태를 대조했습니다. + +## 실행 경로 + +```text +로컬 JVM / 실제 PermissionController·Facade·JPA·Hikari + │ UUID 요청 키 + 합성 문서 단건 권한 회수, 동시성 2 + ▼ +IAP 연결 → 격리 OpenProxy → 현재 Primary의 삭제·캐시·Outbox·요청 원장 + │ │ + │ └─ strict / on → 동기 후보 ACK 후 204 + └─ 읽기 전용 역할 어댑터 → native Patroni의 fresh scope/역할 + +외부 JSONL 원장: 시각·논리 표본·키·응답·지연을 실제 응답마다 기록 + → 시험 컨테이너 강제 종료 → 자동 승격/재가입 관측 + → 503은 같은 키 재시도 → 현재 Primary에서 실제 키별 대조 +``` + +단일 JVM의 MVC 호출이며 공개 HTTP 로그인·브라우저·LB·실제 모델 경로는 포함하지 않습니다. DB와 OpenProxy는 실제 라이선스 호스트의 별도 컨테이너·데이터·포트·DCS namespace입니다. 표의 node 별칭은 실제 인프라 이름이 아닙니다. + +## 원본 증거 + +각 디렉터리에 `app.log`, 실제 응답 `requests.jsonl`, 장애 제어 `control.jsonl`, `measurement.json`, `summary.txt`, 독립 `key-reconciliation.log`가 있습니다. 원시 회차를 덮어쓰지 않았습니다. + +- [지정형 Standby](designated-standby-run05/requests.jsonl) · [키 대조](designated-standby-run05/key-reconciliation.log) +- [quorum Standby](quorum-standby-run01/requests.jsonl) · [키 대조](quorum-standby-run01/key-reconciliation.log) +- [지정형 Primary](designated-primary-run01/requests.jsonl) · [키 대조](designated-primary-run01/key-reconciliation.log) +- [quorum Primary](quorum-primary-run01/requests.jsonl) · [키 대조](quorum-primary-run01/key-reconciliation.log) + +## 준비 실패와 재시험 + +| 실행 | 관측 | 처리 | +| --- | --- | --- | +| preflight JDBC 01–04 | native DB 경로는 읽혔으나 원본 Proxy 연결 실패 | 로그 폴더 권한 문제를 분리한 뒤 native 역할 parser 결함 확인 | +| [preflight JDBC 05](preflight-jdbc-run05.log) | 어댑터 경유 두 Proxy에서 ANY 1·Primary·streaming 2대 확인 | 본 비교로 진행 | +| [designated-standby-run01](designated-standby-run01/control.jsonl) | streaming 1대라 사전 기준 실패 | 부하·장애 주입 미실행, SQL readiness 대기 보완 | +| [designated-standby-run02](designated-standby-run02/control.jsonl) | 강제 종료 후 시험 socket 잠금이 남아 재기동 실패 | 정지된 시험 컨테이너의 전용 socket만 정리 | +| [designated-standby-run03](designated-standby-run03/app.log) | WAN 스키마 초기화·fixture 준비가 180초 준비 제한 초과 | 장애 주입 미실행. 뒤늦게 시작된 부분 응답 10개는 모두 204였으나 시험을 중단했으며 최종 4회 집계에서 제외. 사전 배치 준비로 분리 | +| [designated-standby-run04](designated-standby-run04/app.log) | 48개 204 뒤 검증용 autocommit 조회가 standby에 도착해 Primary 단언 실패 | 최종 대조도 read-write Transaction으로 수정. [실패 회차 키 감사](designated-standby-run04/post-failure-key-audit.log)는 통과 판정에 재사용하지 않음 | +| designated-standby-run05 | 수정된 동일 시험과 독립 키 감사 통과 | 최종 집계에 사용 | + +추가 준비 과정에는 기존 RSA SSH 인증 실패, 임시 만료 키 사용, Docker commit 출력 형식/시간 제한, 읽기 전용 패키지의 로그 경로 준비 문제가 있었습니다. 초기 이미지 캡처 1회에는 Docker 기본 pause가 사용됐고 이후에는 pause=false를 적용했습니다. 공개 HTTP 영향은 계측하지 않았습니다. 공개 DB 설정 hash는 모든 제어 단계에서 일치했습니다. + +## 재실행 + +1. 정식 사용 가능한 기존 라이선스 호스트에 별도 시험 데이터·DCS namespace·포트로 3노드를 준비합니다. 공급사 파일·라이선스·실제 인프라 설정은 이 저장소에 포함하지 않습니다. +2. `patroni_role_compatibility.py`를 시험 scope와 private IPv4에만 연결합니다. 시험 OpenProxy의 Patroni 조회 포트는 해당 어댑터를 가리켜야 합니다. +3. 전용 루프백 JDBC URL·별도 migration URL·비공개 접속 설정과 `PERMISSION_HA_RUN_DIR`, `PERMISSION_HA_MODE`, `PERMISSION_HA_SCENARIO`를 준비하고 `./backend/gradlew -p backend permissionHaTest`를 실행합니다. Task는 전용 loopback Proxy URL만 허용합니다. +4. 실행 디렉터리의 `ready` 뒤에 시험 컨테이너만 중단하고, 실제 승격·재가입 관측 뒤 `fault.done`을 기록합니다. 제어 시각은 별도 JSONL에 남겨야 합니다. +5. 같은 JDBC 드라이버로 `PermissionLedgerReconcile.java`를 실행해 실제 외부 요청 키를 다시 대조합니다. 호스트 접속·자원 생성 제어 프로필은 비공개로 분리했습니다. + +## 수동 검토 뒤 실행 제한 보완 + +- Flyway·fixture 준비 주소도 전용 loopback 포트만 허용하고, 지원하는 모드·시나리오를 제한했습니다. 이전 응답 원장이나 완료 표식이 있는 디렉터리는 재사용하지 않습니다. +- [migration 주소 차단](guard-migration-run01.jsonl), [이전 원장 보호](guard-ledger-run01.jsonl) 두 목적별 검사에서 예상 사전 조건 오류로 실행이 중단되고 원장이 변하지 않았습니다. 이는 의도한 exit 1을 통과 기준으로 한 검사이며 DB 장애 회차 실패가 아닙니다. +- 이 사전 조건·integration 태그 보완 후 GCP 4회차를 다시 실행하지 않았습니다. 위 표는 명시한 `bb3c735` 실행 그대로입니다. + +## 운영 판단과 검증 한계 + +- 이번 조건에서는 quorum의 뚜렷한 우위를 관측하지 못했습니다. 한 Standby 상실은 두 모드 모두 48개 204를 유지했고 Primary 상실은 두 모드 모두 9개 503과 같은 키 재시도로 수렴했습니다. +- 원본 OpenProxy 역할 호환 문제가 있으므로 **현재 공개 클러스터의 동기/quorum 전환은 보류**합니다. 동기 확정 기능 기본값 false이며 운영 설정·배포를 이 시험으로 바꾸지 않았습니다. +- 실제 OpenSQL 자동 승격을 시험했지만 후보 지정 수동 전환, 네트워크 분할, 임의 복제 지연, 동기 후보 전체 상실은 이 GCP 비교 범위 밖입니다. 사전 패키지 읽기에서 확인한 #3310 핵심 패치 부재를 해결했다고 주장하지 않습니다. +- 로컬 시험의 COMMIT 경고·응답 유실 모델과 이번 GCP Primary 중단을 구분합니다. GCP에서 COMMIT 후 HTTP 응답만 차단한 별도 회차는 실행하지 않았습니다. +- 로컬에서는 상세·본문·파일·새 검색 MVC 응답을 검증했습니다. GCP에서는 회수·최종 인가·원장 대조를 검증했습니다. 실제 모델·검색 품질·공개 로그인·GCS는 포함하지 않았습니다. +- 기존 VM의 물리 자원을 공유하고 IAP 지연이 포함됐습니다. 전역 RPO=0·공개 서비스 전체 RTO·성능 SLA·공급사 공식 지원으로 확장하지 않습니다. +- 시간 정의 보완은 원시 원장에 동일 규칙을 적용한 사후 집계이며 원시 `measurement.json`의 fault-requested 기준 값도 보존했습니다. + +## 정리 확인 + +[정리 원장](cleanup.jsonl)에 실제 작업 결과를 시간순으로 보존했습니다. + +- 격리 GCP 컨테이너 3개·전용 데이터 경로 3개·확인된 시험 이미지 3개를 제거했습니다. 전용 DCS 키는 0개입니다. +- 로컬 fixture 컨테이너 3개와 전용 Docker 네트워크 1개를 제거했습니다. +- 30분 임시 OS Login 키는 이미 만료·부재 상태였고 일치하는 키 0개를 다시 확인했습니다. 로컬 개인키·비밀 파일을 제거하고 전용 SSH 제어 연결 3개를 종료했습니다. 기존 개인키·터널은 유지했습니다. +- 공개 클러스터는 최종 조회에서 leader 1대·streaming replica 2대, 기존 비동기 설정으로 확인했습니다. 원본 설정 hash도 3대 모두 같았습니다. 기존 VM은 정지하지 않았습니다. +- 생성 경위가 확정되지 않은 이전 이미지 캐시는 broad prune하지 않았습니다. 따라서 호스트의 모든 이미지 캐시가 0개라고 주장하지 않습니다. diff --git a/docs/test-results/opensql-sync-quorum-20261017/aggregate.json b/docs/test-results/opensql-sync-quorum-20261017/aggregate.json new file mode 100644 index 00000000..784ea8fe --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/aggregate.json @@ -0,0 +1,107 @@ +{ + "calculation": "원시 원장은 보존. 동일 규칙으로 사후 집계한 시간 지표이며 공개 전체 RTO가 아님.", + "runs": [ + { + "run_id": "designated-standby-run05", + "mode": "designated", + "scenario": "standby-loss", + "initial_requests": 48, + "initial_204": 48, + "initial_unconfirmed": 0, + "retry_requests": 0, + "initial_status_counts": { + "204": 48 + }, + "max_initial_204_gap_seconds": 4.810205, + "write_resume_after_confirmed_fault_seconds": 3.526817, + "promotion_after_confirmed_fault_seconds": null, + "initial_unknown": 0, + "final_receipts": 48, + "final_permissions": 0, + "forbidden_content": 0, + "auth_unavailable": 0, + "p50_ms": 2803.802209, + "p95_ms": 2906.397625, + "hikari_max_active": 2, + "hikari_max_pending": 0, + "key_reconciliation_passed": true + }, + { + "run_id": "quorum-standby-run01", + "mode": "quorum", + "scenario": "standby-loss", + "initial_requests": 48, + "initial_204": 48, + "initial_unconfirmed": 0, + "retry_requests": 0, + "initial_status_counts": { + "204": 48 + }, + "max_initial_204_gap_seconds": 4.743453, + "write_resume_after_confirmed_fault_seconds": 3.47442, + "promotion_after_confirmed_fault_seconds": null, + "initial_unknown": 0, + "final_receipts": 48, + "final_permissions": 0, + "forbidden_content": 0, + "auth_unavailable": 0, + "p50_ms": 2796.313417, + "p95_ms": 2920.39875, + "hikari_max_active": 2, + "hikari_max_pending": 0, + "key_reconciliation_passed": true + }, + { + "run_id": "designated-primary-run01", + "mode": "designated", + "scenario": "primary-loss", + "initial_requests": 48, + "initial_204": 39, + "initial_unconfirmed": 9, + "retry_requests": 9, + "initial_status_counts": { + "204": 39, + "503": 9 + }, + "max_initial_204_gap_seconds": 27.001096, + "write_resume_after_confirmed_fault_seconds": 26.066879, + "promotion_after_confirmed_fault_seconds": 20.385079, + "initial_unknown": 9, + "final_receipts": 48, + "final_permissions": 0, + "forbidden_content": 0, + "auth_unavailable": 2, + "p50_ms": 2807.336291, + "p95_ms": 5018.840625, + "hikari_max_active": 2, + "hikari_max_pending": 2, + "key_reconciliation_passed": true + }, + { + "run_id": "quorum-primary-run01", + "mode": "quorum", + "scenario": "primary-loss", + "initial_requests": 48, + "initial_204": 39, + "initial_unconfirmed": 9, + "retry_requests": 9, + "initial_status_counts": { + "204": 39, + "503": 9 + }, + "max_initial_204_gap_seconds": 26.969831, + "write_resume_after_confirmed_fault_seconds": 25.90207, + "promotion_after_confirmed_fault_seconds": 18.695327, + "initial_unknown": 9, + "final_receipts": 48, + "final_permissions": 0, + "forbidden_content": 0, + "auth_unavailable": 2, + "p50_ms": 2804.792959, + "p95_ms": 5019.69925, + "hikari_max_active": 2, + "hikari_max_pending": 2, + "key_reconciliation_passed": true + } + ] +} diff --git a/docs/test-results/opensql-sync-quorum-20261017/cleanup.jsonl b/docs/test-results/opensql-sync-quorum-20261017/cleanup.jsonl new file mode 100644 index 00000000..f2b091ed --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/cleanup.jsonl @@ -0,0 +1,14 @@ +{"at_kst": "2026-10-17T07:01:11.001497+09:00", "run_id": "final-cleanup-run01", "event": "start", "purpose": "표식·경로가 확인된 시험 자원과 임시 접속만 제거", "location": "기존 라이선스 호스트 3대 및 로컬", "criteria": "시험 컨테이너·데이터·DCS 제거, 공개 설정 유지, 임시 키 제거, 원래 VM 유지"} +{"at_kst": "2026-10-17T07:01:11.999211+09:00", "run_id": "final-cleanup-run01", "event": "test-stop", "result": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T22:01:11.929943+00:00", "production_config_unchanged": true}} +{"at_kst": "2026-10-17T07:01:13.048249+09:00", "run_id": "final-cleanup-run01", "event": "test-stop", "result": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T22:01:12.904541+00:00", "production_config_unchanged": true}} +{"at_kst": "2026-10-17T07:01:13.988489+09:00", "run_id": "final-cleanup-run01", "event": "test-stop", "result": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T22:01:13.937790+00:00", "production_config_unchanged": true}} +{"at_kst": "2026-10-17T07:01:15.212188+09:00", "run_id": "final-cleanup-run01", "event": "remote-cleanup", "result": {"node_alias": "node1", "action": "final-cleanup", "container_removed": true, "test_root_removed": true, "known_test_image_removed": true, "isolated_dcs_keys_remaining": 0, "production_config_unchanged": true, "public_state": "running", "public_role": "primary", "public_sync_mode": false, "public_sync_strict": false, "at_utc": "2026-10-16T22:01:15.164346+00:00", "ambiguous_image_caches_not_pruned": true}} +{"at_kst": "2026-10-17T07:01:16.474219+09:00", "run_id": "final-cleanup-run01", "event": "remote-cleanup", "result": {"node_alias": "node2", "action": "final-cleanup", "container_removed": true, "test_root_removed": true, "known_test_image_removed": true, "isolated_dcs_keys_remaining": 0, "production_config_unchanged": true, "public_state": "running", "public_role": "replica", "public_sync_mode": false, "public_sync_strict": false, "at_utc": "2026-10-16T22:01:16.422638+00:00", "ambiguous_image_caches_not_pruned": true}} +{"at_kst": "2026-10-17T07:01:17.832914+09:00", "run_id": "final-cleanup-run01", "event": "remote-cleanup", "result": {"node_alias": "node3", "action": "final-cleanup", "container_removed": true, "test_root_removed": true, "known_test_image_removed": true, "isolated_dcs_keys_remaining": 0, "production_config_unchanged": true, "public_state": "running", "public_role": "replica", "public_sync_mode": false, "public_sync_strict": false, "at_utc": "2026-10-16T22:01:17.787285+00:00", "ambiguous_image_caches_not_pruned": true}} +{"at_kst": "2026-10-17T07:02:18.879463+09:00", "run_id": "final-cleanup-run01", "event": "local-fixture-cleanup", "procedure": "permission_isolated_fixture.py down", "containers_removed": 3, "network_removed": 1, "observed_exit": 0} +{"at_kst": "2026-10-17T07:02:19.621904+09:00", "run_id": "final-cleanup-run01", "event": "public-final-health", "result": {"node_alias": "node1", "action": "public-final-health", "member_count": 3, "roles": {"leader": 1, "replica": 2}, "states": {"running": 1, "streaming": 2}, "at_utc": "2026-10-16T22:02:19.583847+00:00"}} +{"at_kst": "2026-10-17T07:02:21.146498+09:00", "run_id": "final-cleanup-run01", "event": "temporary-os-login-key", "outcome": "already_absent_or_expired", "matching_key_remaining": 0, "permanent_key_untouched": true} +{"at_kst": "2026-10-17T07:02:21.151867+09:00", "run_id": "final-cleanup-run01", "event": "temporary-channel-close", "node_alias": "node1", "exit": 0} +{"at_kst": "2026-10-17T07:02:21.156162+09:00", "run_id": "final-cleanup-run01", "event": "temporary-channel-close", "node_alias": "node2", "exit": 0} +{"at_kst": "2026-10-17T07:02:21.160459+09:00", "run_id": "final-cleanup-run01", "event": "temporary-channel-close", "node_alias": "node3", "exit": 0} +{"at_kst": "2026-10-17T07:02:21.161623+09:00", "run_id": "final-cleanup-run01", "event": "local-private-state-removed", "secret_file_removed": true, "key_directory_removed": true, "channel_state_removed": true, "original_tunnels_untouched": true, "original_vms_left_running": true} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01-preflight.log b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01-preflight.log new file mode 100644 index 00000000..6d505538 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01-preflight.log @@ -0,0 +1,13 @@ +실행 ID: designated-primary-run01-preflight +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T06:39:44.658931+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=2 +EVIDENCE replica_alias=permission-node2 sync_state=sync +EVIDENCE replica_alias=permission-node3 sync_state=async +EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names="test-standby" +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/app.log b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/app.log new file mode 100644 index 00000000..c10453e8 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/app.log @@ -0,0 +1,15 @@ +{"실행_ID": "designated-primary-run01-app", "목적": "실제 격리 OpenSQL·OpenProxy의 권한 회수 HA 계약", "시작_KST": "2026-10-17T06:39:49.756391+09:00", "위치": "로컬 JVM → IAP → 격리 GCP OpenSQL·OpenProxy·역할 어댑터", "명령": "./backend/gradlew -p backend permissionHaTest --console=plain", "성공_기준": "실제 커밋 경계·원장·인가 단언 전부 통과; 실패·미실행을 구분"} +> Task :compileJava UP-TO-DATE +> Task :processResources UP-TO-DATE +> Task :classes UP-TO-DATE +> Task :compileTestJava UP-TO-DATE +> Task :processTestResources UP-TO-DATE +> Task :testClasses UP-TO-DATE +> Task :permissionHaTest +> Task :permissionHaTest +EVIDENCE 2026-10-17T06:40:06.401402+09:00 mode=designated scenario=primary-loss phase=fixture-start samples=48 +EVIDENCE 2026-10-17T06:40:08.188685+09:00 mode=designated scenario=primary-loss phase=fixture-ready samples=48 +EVIDENCE 2026-10-17T06:42:45.731869+09:00 mode=designated scenario=primary-loss phase=summary initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2807.336291 p95_ms=5018.840625 hikari_max_active=2 hikari_max_pending=2 +OpenSqlSynchronousPolicyComparisonTest > 동일 회수 부하 중 장애가 있어도 확정 요청이 보존되고 불명 요청은 같은 키로 수렴한다 PASSED +BUILD SUCCESSFUL in 2m 56s +{"종료_KST": "2026-10-17T06:42:45.850975+09:00", "exit_code": 0} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/control.jsonl new file mode 100644 index 00000000..1b5db9e8 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/control.jsonl @@ -0,0 +1,36 @@ +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:14.737004+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "bb3c73521155a64e9675a28a32c9f673f9a47e0f"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:14.737359+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:15.622307+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:39:15.568028+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:16.547044+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:39:16.490113+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:17.443004+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:39:17.391483+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:18.263149+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T21:39:18.205787+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:19.132464+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:39:19.074055+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:20.034293+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:39:19.977348+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:20.857023+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:39:20.810558+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:21.814087+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:39:21.723090+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:26.724593+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:39:26.628858+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:34.724938+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "start", "completed": true, "at_utc": "2026-10-16T21:39:34.567424+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:40.089274+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "restore-schema", "completed": true, "at_utc": "2026-10-16T21:39:40.036251+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:40.906480+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:41.735661+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:42.515049+09:00", "phase": "remote", "node_alias": "node3", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:43.377692+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:44.219199+09:00", "phase": "remote", "node_alias": "node3", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:44.609306+09:00", "phase": "dynamic-observed", "configuration": {"synchronous_mode": true, "synchronous_mode_strict": true, "synchronous_node_count": 1}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:44.609499+09:00", "phase": "reset-ready", "primary": "node1", "replicas": 2, "mode": "designated", "strict": true, "node_count": 1} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:49.717389+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:49.717616+09:00", "phase": "preflight", "evidence": "EVIDENCE streaming_replicas=2"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:49.717636+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node2 sync_state=sync"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:49.717656+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node3 sync_state=async"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:49.717669+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:39:49.717684+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:40:16.033181+09:00", "phase": "fault-requested", "target": "node1", "scenario": "primary-loss"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:40:16.945184+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:40:16.893416+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:40:16.945357+09:00", "phase": "fault-confirmed", "target": "node1"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:40:37.330436+09:00", "phase": "automatic-promotion", "new_primary": "node2"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:40:38.386809+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:40:38.249242+00:00", "production_config_unchanged": true}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:40:39.283856+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:40:43.326804+09:00", "phase": "rejoined", "target": "node1", "role": "replica"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:42:45.883242+09:00", "phase": "app-finished", "exit_code": 0} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:42:47.780100+09:00", "phase": "key-reconciliation", "evidence": "EVIDENCE ledger_unique_keys=48 initial_204_keys=39 acknowledged_preserved=39 resolved_keys=48 remaining_permissions=0"} +{"run_id": "designated-primary-run01", "at_kst": "2026-10-17T06:42:47.783894+09:00", "phase": "summary", "measurement": {"run_id": "designated-primary-run01", "mode": "designated", "scenario": "primary-loss", "initial_requests": 48, "initial_204": 39, "initial_unconfirmed": 9, "retry_requests": 9, "max_success_gap_seconds": 27.001096, "first_204_after_fault_requested_seconds": 26.979081, "app_contract": "initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2807.336291 p95_ms=5018.840625 hikari_max_active=2 hikari_max_pending=2"}} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/key-reconciliation.log b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/key-reconciliation.log new file mode 100644 index 00000000..923f88bd --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/key-reconciliation.log @@ -0,0 +1,4 @@ +목적: 외부 원장의 실제 요청 키별 성공 보존·불명 해소 대조 +위치: 현재 Primary / 같은 격리 OpenProxy 경로 +EVIDENCE ledger_unique_keys=48 initial_204_keys=39 acknowledged_preserved=39 resolved_keys=48 remaining_permissions=0 +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/measurement.json b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/measurement.json new file mode 100644 index 00000000..1b6698fb --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/measurement.json @@ -0,0 +1,12 @@ +{ + "run_id": "designated-primary-run01", + "mode": "designated", + "scenario": "primary-loss", + "initial_requests": 48, + "initial_204": 39, + "initial_unconfirmed": 9, + "retry_requests": 9, + "max_success_gap_seconds": 27.001096, + "first_204_after_fault_requested_seconds": 26.979081, + "app_contract": "initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2807.336291 p95_ms=5018.840625 hikari_max_active=2 hikari_max_pending=2" +} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/requests.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/requests.jsonl new file mode 100644 index 00000000..cc5c0246 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/requests.jsonl @@ -0,0 +1,57 @@ +{"at":"2026-10-17T06:40:11.345822+09:00","sample":0,"key":"8ee5ff30-7f9d-464a-a4bc-5a3924b17c00","retry":false,"status":204,"latency_ms":3156.189625} +{"at":"2026-10-17T06:40:11.349312+09:00","sample":1,"key":"df1bd55d-8eb5-46d7-b40a-ef9b53a8fb11","retry":false,"status":204,"latency_ms":3156.191417} +{"at":"2026-10-17T06:40:16.010694+09:00","sample":2,"key":"8614f22e-01c9-47f3-bb60-3050d3c6f0dc","retry":false,"status":204,"latency_ms":2795.17} +{"at":"2026-10-17T06:40:16.011140+09:00","sample":3,"key":"bfdd4a0d-d6d8-4ec0-84dc-19639a4116b9","retry":false,"status":204,"latency_ms":2795.066708} +{"at":"2026-10-17T06:40:17.448359+09:00","sample":5,"key":"e25defab-5048-43b7-bdfb-611a0b18e169","retry":false,"status":503,"latency_ms":407.853375} +{"at":"2026-10-17T06:40:22.059315+09:00","sample":4,"key":"d089195e-4c5b-4b27-9a13-843db510d86c","retry":false,"status":503,"latency_ms":5018.840625} +{"at":"2026-10-17T06:40:22.461936+09:00","sample":6,"key":"6365ca85-088c-4e8b-919c-73cf5ddf2b4f","retry":false,"status":503,"latency_ms":5013.36275} +{"at":"2026-10-17T06:40:27.072276+09:00","sample":7,"key":"1ced8794-cf88-4fb1-9c7e-ea5cc55be7ab","retry":false,"status":503,"latency_ms":5012.549625} +{"at":"2026-10-17T06:40:27.475277+09:00","sample":8,"key":"8770f7ab-9d9b-40f1-bab4-f7939e546e15","retry":false,"status":503,"latency_ms":5012.6155} +{"at":"2026-10-17T06:40:32.090139+09:00","sample":9,"key":"db4057e2-2fdf-4ea1-9c21-6a3862dcb261","retry":false,"status":503,"latency_ms":5014.183041} +{"at":"2026-10-17T06:40:32.486344+09:00","sample":10,"key":"eec26e59-2fb2-4e9e-aa18-db21d886268b","retry":false,"status":503,"latency_ms":5010.475917} +{"at":"2026-10-17T06:40:37.101550+09:00","sample":11,"key":"ac98d812-16b4-41d4-a262-282280b76a07","retry":false,"status":503,"latency_ms":5010.681125} +{"at":"2026-10-17T06:40:37.492234+09:00","sample":12,"key":"87ff4af3-93d5-49c5-888a-2b506dbe4a5d","retry":false,"status":503,"latency_ms":5005.202958} +{"at":"2026-10-17T06:40:43.012236+09:00","sample":13,"key":"a359fb76-7b60-48ab-be60-33abce66e259","retry":false,"status":204,"latency_ms":5910.179667} +{"at":"2026-10-17T06:40:44.022070+09:00","sample":14,"key":"3eb2770d-e12d-4a4b-8318-792cde4454a5","retry":false,"status":204,"latency_ms":6529.71325} +{"at":"2026-10-17T06:40:47.610429+09:00","sample":15,"key":"4b3b07be-2591-4328-98d9-76e2efb5d4d8","retry":false,"status":204,"latency_ms":2799.892166} +{"at":"2026-10-17T06:40:48.608690+09:00","sample":16,"key":"8a15c3b2-0d1b-4008-83ea-20b66cd5800b","retry":false,"status":204,"latency_ms":2794.964708} +{"at":"2026-10-17T06:40:52.210048+09:00","sample":17,"key":"f345916e-f82b-4a5a-b232-e757d5bcf3ba","retry":false,"status":204,"latency_ms":2808.79975} +{"at":"2026-10-17T06:40:53.210645+09:00","sample":18,"key":"a514bb14-1a80-4bef-ba9e-e9f2a646301c","retry":false,"status":204,"latency_ms":2804.645541} +{"at":"2026-10-17T06:40:56.813890+09:00","sample":19,"key":"d5731404-771e-4e5d-8db1-0a09eee62ed6","retry":false,"status":204,"latency_ms":2811.949208} +{"at":"2026-10-17T06:40:57.806918+09:00","sample":20,"key":"e62bb9db-4823-44e7-a700-d4c59499de7c","retry":false,"status":204,"latency_ms":2801.146792} +{"at":"2026-10-17T06:41:01.407453+09:00","sample":21,"key":"91f97497-be0d-43dc-9d39-7ebab80ef451","retry":false,"status":204,"latency_ms":2800.802} +{"at":"2026-10-17T06:41:02.405835+09:00","sample":22,"key":"e342816e-8762-4c67-af8f-967c1e214651","retry":false,"status":204,"latency_ms":2802.921333} +{"at":"2026-10-17T06:41:06.107475+09:00","sample":23,"key":"cc7aa2e3-9183-446f-9d3d-cea1d2077bd5","retry":false,"status":204,"latency_ms":2838.13325} +{"at":"2026-10-17T06:41:07.107785+09:00","sample":24,"key":"b8441912-ea5e-4711-9415-dc97b8370cf5","retry":false,"status":204,"latency_ms":2807.336291} +{"at":"2026-10-17T06:41:10.698332+09:00","sample":25,"key":"279f5b85-73e6-46c5-b374-09b8654ac941","retry":false,"status":204,"latency_ms":2790.724458} +{"at":"2026-10-17T06:41:11.712395+09:00","sample":26,"key":"5bf531bf-96b6-4d01-b298-5b2a019ba701","retry":false,"status":204,"latency_ms":2815.009833} +{"at":"2026-10-17T06:41:15.299884+09:00","sample":27,"key":"1d0ca1b2-974a-4034-ad5e-28e9a1aa1019","retry":false,"status":204,"latency_ms":2795.38225} +{"at":"2026-10-17T06:41:16.309660+09:00","sample":28,"key":"4f402303-0158-4336-97bd-c1d7b7be1d60","retry":false,"status":204,"latency_ms":2805.3045} +{"at":"2026-10-17T06:41:19.906616+09:00","sample":29,"key":"51cbd7ee-6a17-4c23-8766-f905a41676e5","retry":false,"status":204,"latency_ms":2801.6975} +{"at":"2026-10-17T06:41:20.905154+09:00","sample":30,"key":"954de8bb-50ce-476f-ab4e-eddf57cdd156","retry":false,"status":204,"latency_ms":2799.708} +{"at":"2026-10-17T06:41:24.529761+09:00","sample":31,"key":"62de29c6-394d-418a-abd8-59f9ad2bc8d1","retry":false,"status":204,"latency_ms":2821.463833} +{"at":"2026-10-17T06:41:25.519901+09:00","sample":32,"key":"5ede2843-7606-46a4-bb1a-07205e18d174","retry":false,"status":204,"latency_ms":2809.021458} +{"at":"2026-10-17T06:41:29.100869+09:00","sample":33,"key":"aad27947-3666-423b-ae1f-61d11d000377","retry":false,"status":204,"latency_ms":2795.376917} +{"at":"2026-10-17T06:41:30.110230+09:00","sample":34,"key":"5490e313-bdfc-4e00-93e7-ed9c3fdee354","retry":false,"status":204,"latency_ms":2805.41475} +{"at":"2026-10-17T06:41:33.708825+09:00","sample":35,"key":"7c0d997e-e36e-4fa9-b8ff-f733a9165e62","retry":false,"status":204,"latency_ms":2803.168334} +{"at":"2026-10-17T06:41:34.703062+09:00","sample":36,"key":"33841a6d-50e2-42b1-bb1d-a1f19905b2be","retry":false,"status":204,"latency_ms":2797.815167} +{"at":"2026-10-17T06:41:38.305746+09:00","sample":37,"key":"f0c6621a-964b-4159-b850-ef5c6e2a30fa","retry":false,"status":204,"latency_ms":2800.491167} +{"at":"2026-10-17T06:41:39.301386+09:00","sample":38,"key":"eb9c4519-a707-4624-b059-f6baf53085f8","retry":false,"status":204,"latency_ms":2801.078} +{"at":"2026-10-17T06:41:42.910919+09:00","sample":39,"key":"16a40143-0c18-4e3b-82b5-123e9db426de","retry":false,"status":204,"latency_ms":2822.27725} +{"at":"2026-10-17T06:41:43.907125+09:00","sample":40,"key":"84bdd342-5d6e-4663-8f47-b87851a6bfbd","retry":false,"status":204,"latency_ms":2802.495292} +{"at":"2026-10-17T06:41:47.513659+09:00","sample":41,"key":"eb3ef5cf-069d-425c-8fe8-302e84240dae","retry":false,"status":204,"latency_ms":2810.500417} +{"at":"2026-10-17T06:41:48.500481+09:00","sample":42,"key":"2ff03468-3c38-46fb-bb39-98dd8b32f9dc","retry":false,"status":204,"latency_ms":2809.777834} +{"at":"2026-10-17T06:41:52.104170+09:00","sample":43,"key":"392a0f94-060d-4f5b-980c-3beebc387fd3","retry":false,"status":204,"latency_ms":2810.973667} +{"at":"2026-10-17T06:41:53.112642+09:00","sample":44,"key":"4b240218-2aef-440e-a0f8-155aaced3b13","retry":false,"status":204,"latency_ms":2813.706334} +{"at":"2026-10-17T06:41:56.709625+09:00","sample":45,"key":"0fadfc22-a24d-43e0-90b5-935a29cf124c","retry":false,"status":204,"latency_ms":2803.898333} +{"at":"2026-10-17T06:41:57.709261+09:00","sample":46,"key":"3e1d2870-3c23-4967-8f39-e978388d1d83","retry":false,"status":204,"latency_ms":2805.480791} +{"at":"2026-10-17T06:42:01.310724+09:00","sample":47,"key":"ebdecd26-d604-4c74-bcd8-869b12bf0546","retry":false,"status":204,"latency_ms":2808.701834} +{"at":"2026-10-17T06:42:05.942016+09:00","sample":5,"key":"e25defab-5048-43b7-bdfb-611a0b18e169","retry":true,"status":204,"latency_ms":2791.794084} +{"at":"2026-10-17T06:42:10.508292+09:00","sample":4,"key":"d089195e-4c5b-4b27-9a13-843db510d86c","retry":true,"status":204,"latency_ms":2781.263875} +{"at":"2026-10-17T06:42:15.110445+09:00","sample":6,"key":"6365ca85-088c-4e8b-919c-73cf5ddf2b4f","retry":true,"status":204,"latency_ms":2807.491792} +{"at":"2026-10-17T06:42:19.699295+09:00","sample":7,"key":"1ced8794-cf88-4fb1-9c7e-ea5cc55be7ab","retry":true,"status":204,"latency_ms":2792.741625} +{"at":"2026-10-17T06:42:24.307651+09:00","sample":8,"key":"8770f7ab-9d9b-40f1-bab4-f7939e546e15","retry":true,"status":204,"latency_ms":2809.01625} +{"at":"2026-10-17T06:42:28.905119+09:00","sample":9,"key":"db4057e2-2fdf-4ea1-9c21-6a3862dcb261","retry":true,"status":204,"latency_ms":2802.769541} +{"at":"2026-10-17T06:42:33.518165+09:00","sample":10,"key":"eec26e59-2fb2-4e9e-aa18-db21d886268b","retry":true,"status":204,"latency_ms":2811.826833} +{"at":"2026-10-17T06:42:38.560644+09:00","sample":11,"key":"ac98d812-16b4-41d4-a262-282280b76a07","retry":true,"status":204,"latency_ms":3256.49775} +{"at":"2026-10-17T06:42:43.112844+09:00","sample":12,"key":"87ff4af3-93d5-49c5-888a-2b506dbe4a5d","retry":true,"status":204,"latency_ms":2772.158167} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/summary.txt b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/summary.txt new file mode 100644 index 00000000..6e1b933c --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-primary-run01/summary.txt @@ -0,0 +1 @@ +initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2807.336291 p95_ms=5018.840625 hikari_max_active=2 hikari_max_pending=2 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run01-preflight.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run01-preflight.log new file mode 100644 index 00000000..bc043696 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run01-preflight.log @@ -0,0 +1,10 @@ +실행 ID: designated-standby-run01-preflight +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T05:59:34.761388+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=1 +EVIDENCE alias=direct-primary error_class=IllegalStateException sqlstate=unavailable +exit_code=1 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run01/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run01/control.jsonl new file mode 100644 index 00000000..45a7184e --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run01/control.jsonl @@ -0,0 +1,19 @@ +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:08.926234+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "e798b4e32e888548e6ff8c486dbc99081f94b5d9"} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:08.926423+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:09.814463+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T20:59:09.773969+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:10.695125+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T20:59:10.661930+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:11.717661+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T20:59:11.675016+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:12.503073+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T20:59:12.468094+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:13.362805+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T20:59:13.324801+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:14.191909+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T20:59:14.156079+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:15.037430+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T20:59:14.998129+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:15.990030+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T20:59:15.952474+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:20.855684+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T20:59:20.812547+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:28.686863+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "start", "completed": true, "at_utc": "2026-10-16T20:59:28.621044+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:33.870450+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18009, "public_proxy_changed": false}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:34.723130+09:00", "phase": "remote", "node_alias": "node3", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18009, "public_proxy_changed": false}} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:34.723263+09:00", "phase": "reset-ready", "primary": "node1", "replicas": 2, "mode": "designated", "strict": true, "node_count": 1} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:36.263291+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:36.263438+09:00", "phase": "preflight", "evidence": "EVIDENCE streaming_replicas=1"} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:36.263464+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary error_class=IllegalStateException sqlstate=unavailable"} +{"run_id": "designated-standby-run01", "at_kst": "2026-10-17T05:59:36.263731+09:00", "phase": "failed", "error_class": "RuntimeError", "category": "jdbc_preflight_failed"} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run02/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run02/control.jsonl new file mode 100644 index 00000000..0ac26ef1 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run02/control.jsonl @@ -0,0 +1,11 @@ +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:23.870844+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "884f4250f7fb6746f86f343fef8fc3ecd4e31fc2"} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:23.871050+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:24.848459+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:11:24.779983+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:25.805989+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:11:25.787219+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:26.710556+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:11:26.697443+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:27.503474+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T21:11:27.492997+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:28.340721+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:11:28.331236+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:29.181373+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:11:29.170483+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:30.023001+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:11:30.010597+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:11:30.989863+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:11:30.967952+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run02", "at_kst": "2026-10-17T06:12:31.210409+09:00", "phase": "failed", "error_class": "RuntimeError", "category": "isolated_role_timeout_node1"} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03-preflight.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03-preflight.log new file mode 100644 index 00000000..ee7994d3 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03-preflight.log @@ -0,0 +1,13 @@ +실행 ID: designated-standby-run03-preflight +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T06:17:18.483805+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=2 +EVIDENCE replica_alias=permission-node2 sync_state=sync +EVIDENCE replica_alias=permission-node3 sync_state=async +EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names="test-standby" +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/app.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/app.log new file mode 100644 index 00000000..6974e882 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/app.log @@ -0,0 +1,14 @@ +{"실행_ID": "designated-standby-run03-app", "목적": "실제 격리 OpenSQL·OpenProxy의 권한 회수 HA 계약", "시작_KST": "2026-10-17T06:17:23.584795+09:00", "위치": "로컬 JVM → IAP → 격리 GCP OpenSQL·OpenProxy·역할 어댑터", "명령": "./backend/gradlew -p backend permissionHaTest --console=plain", "성공_기준": "실제 커밋 경계·원장·인가 단언 전부 통과; 실패·미실행을 구분"} +> Task :compileJava UP-TO-DATE +> Task :processResources UP-TO-DATE +> Task :classes UP-TO-DATE +> Task :compileTestJava +> Task :processTestResources UP-TO-DATE +> Task :testClasses +> Task :permissionHaTest +> Task :permissionHaTest +EVIDENCE 2026-10-17T06:21:03.847711+09:00 mode=designated scenario=standby-loss phase=fixture-start samples=48 +EVIDENCE 2026-10-17T06:22:37.751659+09:00 mode=designated scenario=standby-loss phase=fixture-ready samples=48 +> Task :permissionHaTest FAILED +BUILD FAILED in 5m 36s +{"종료_KST": "2026-10-17T06:23:00.548437+09:00", "exit_code": 1} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/control.jsonl new file mode 100644 index 00000000..a2ac84b9 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/control.jsonl @@ -0,0 +1,26 @@ +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:50.524531+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "884f4250f7fb6746f86f343fef8fc3ecd4e31fc2"} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:50.524723+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:51.312068+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:16:51.241982+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:52.159431+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:16:52.095718+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:53.062602+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:16:52.997910+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:53.864528+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T21:16:53.801503+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:54.703680+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:16:54.637470+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:55.515222+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:16:55.445834+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:56.312263+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:16:56.250801+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:16:57.304267+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:16:57.174955+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:02.208974+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:17:02.109216+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:09.916842+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "start", "completed": true, "at_utc": "2026-10-16T21:17:09.841046+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:14.645409+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:15.502891+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:16.337045+09:00", "phase": "remote", "node_alias": "node3", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:17.221151+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:18.044534+09:00", "phase": "remote", "node_alias": "node3", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:18.431334+09:00", "phase": "dynamic-observed", "configuration": {"synchronous_mode": true, "synchronous_mode_strict": true, "synchronous_node_count": 1}} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:18.431540+09:00", "phase": "reset-ready", "primary": "node1", "replicas": 2, "mode": "designated", "strict": true, "node_count": 1} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:23.548096+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:23.548321+09:00", "phase": "preflight", "evidence": "EVIDENCE streaming_replicas=2"} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:23.548347+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node2 sync_state=sync"} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:23.548370+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node3 sync_state=async"} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:23.548388+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:17:23.548406+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run03", "at_kst": "2026-10-17T06:20:23.614262+09:00", "phase": "failed", "error_class": "RuntimeError", "category": "app_never_ready"} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/requests.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/requests.jsonl new file mode 100644 index 00000000..ad26da57 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run03/requests.jsonl @@ -0,0 +1,10 @@ +{"at":"2026-10-17T06:22:40.637411+09:00","sample":0,"key":"a6b14c36-7ee5-47c0-80c2-7444c7e31664","retry":false,"status":204,"latency_ms":2883.806375} +{"at":"2026-10-17T06:22:40.780583+09:00","sample":1,"key":"47e420f0-71fe-4faf-a496-b7b6ee08ddfc","retry":false,"status":204,"latency_ms":3027.078333} +{"at":"2026-10-17T06:22:45.243801+09:00","sample":2,"key":"1e99aac1-bc6c-44b5-8db9-29f0ede79226","retry":false,"status":204,"latency_ms":2788.1375} +{"at":"2026-10-17T06:22:45.351062+09:00","sample":3,"key":"3875b0be-5e52-41fd-b490-0fefebfff4d7","retry":false,"status":204,"latency_ms":2781.724125} +{"at":"2026-10-17T06:22:49.847192+09:00","sample":4,"key":"ea670529-5f8f-43f9-8527-7e15e341c322","retry":false,"status":204,"latency_ms":2803.0855} +{"at":"2026-10-17T06:22:49.964623+09:00","sample":5,"key":"09806cae-638d-474a-95bd-3ca939e7a6fb","retry":false,"status":204,"latency_ms":2797.705291} +{"at":"2026-10-17T06:22:54.450649+09:00","sample":6,"key":"b1c26233-3c25-47fd-b3d6-00776a010053","retry":false,"status":204,"latency_ms":2811.955416} +{"at":"2026-10-17T06:22:54.549824+09:00","sample":7,"key":"d4be454c-15be-456f-8a00-68785551fd6f","retry":false,"status":204,"latency_ms":2788.99275} +{"at":"2026-10-17T06:22:59.048408+09:00","sample":8,"key":"7530ffc7-f585-421d-ba53-32c62bd5fad0","retry":false,"status":204,"latency_ms":2804.05} +{"at":"2026-10-17T06:22:59.153983+09:00","sample":9,"key":"b5fcae36-3873-4c10-8ded-dccae1ff2957","retry":false,"status":204,"latency_ms":2807.613041} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04-preflight.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04-preflight.log new file mode 100644 index 00000000..b059e8ad --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04-preflight.log @@ -0,0 +1,13 @@ +실행 ID: designated-standby-run04-preflight +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T06:26:30.214270+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=2 +EVIDENCE replica_alias=permission-node2 sync_state=sync +EVIDENCE replica_alias=permission-node3 sync_state=async +EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names="test-standby" +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/app.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/app.log new file mode 100644 index 00000000..deb0fdf5 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/app.log @@ -0,0 +1,15 @@ +{"실행_ID": "designated-standby-run04-app", "목적": "실제 격리 OpenSQL·OpenProxy의 권한 회수 HA 계약", "시작_KST": "2026-10-17T06:26:35.284169+09:00", "위치": "로컬 JVM → IAP → 격리 GCP OpenSQL·OpenProxy·역할 어댑터", "명령": "./backend/gradlew -p backend permissionHaTest --console=plain", "성공_기준": "실제 커밋 경계·원장·인가 단언 전부 통과; 실패·미실행을 구분"} +> Task :compileJava UP-TO-DATE +> Task :processResources UP-TO-DATE +> Task :classes UP-TO-DATE +> Task :compileTestJava UP-TO-DATE +> Task :processTestResources UP-TO-DATE +> Task :testClasses UP-TO-DATE +> Task :permissionHaTest +> Task :permissionHaTest +EVIDENCE 2026-10-17T06:26:51.796062+09:00 mode=designated scenario=standby-loss phase=fixture-start samples=48 +EVIDENCE 2026-10-17T06:26:53.559402+09:00 mode=designated scenario=standby-loss phase=fixture-ready samples=48 +OpenSqlSynchronousPolicyComparisonTest > 동일 회수 부하 중 장애가 있어도 확정 요청이 보존되고 불명 요청은 같은 키로 수렴한다 FAILED +> Task :permissionHaTest FAILED +BUILD FAILED in 2m 9s +{"종료_KST": "2026-10-17T06:28:44.799317+09:00", "exit_code": 1} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/control.jsonl new file mode 100644 index 00000000..a980779d --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/control.jsonl @@ -0,0 +1,35 @@ +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:00.402504+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "39ac0f0e46271ac724e3a818d8043456b56610c7"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:00.402701+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:01.320292+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:26:01.256205+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:02.306643+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:26:02.231415+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:03.191797+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:26:03.131965+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:03.991586+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T21:26:03.931173+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:04.838359+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:26:04.779249+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:05.708570+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:26:05.648562+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:06.554161+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:26:06.493064+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:07.591856+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:26:07.433913+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:12.460533+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:26:12.393734+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:20.401960+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "start", "completed": true, "at_utc": "2026-10-16T21:26:20.337771+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:25.582014+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "restore-schema", "completed": true, "at_utc": "2026-10-16T21:26:25.526693+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:26.382114+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:27.230103+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:28.113668+09:00", "phase": "remote", "node_alias": "node3", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:28.954214+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:29.782302+09:00", "phase": "remote", "node_alias": "node3", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:30.167843+09:00", "phase": "dynamic-observed", "configuration": {"synchronous_mode": true, "synchronous_mode_strict": true, "synchronous_node_count": 1}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:30.167989+09:00", "phase": "reset-ready", "primary": "node1", "replicas": 2, "mode": "designated", "strict": true, "node_count": 1} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:35.251136+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:35.251318+09:00", "phase": "preflight", "evidence": "EVIDENCE streaming_replicas=2"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:35.251342+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node2 sync_state=sync"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:35.251366+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node3 sync_state=async"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:35.251380+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:26:35.251395+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:27:01.563519+09:00", "phase": "fault-requested", "target": "node2", "scenario": "standby-loss"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:27:02.576575+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:27:02.507929+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:27:02.576751+09:00", "phase": "fault-confirmed", "target": "node2"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:27:11.591757+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:27:11.526691+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:27:12.468273+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:27:13.445801+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:27:15.107773+09:00", "phase": "rejoined", "target": "node2", "role": "replica"} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:28:44.856355+09:00", "phase": "app-finished", "exit_code": 1} +{"run_id": "designated-standby-run04", "at_kst": "2026-10-17T06:28:44.856741+09:00", "phase": "failed", "error_class": "RuntimeError", "category": "app_contract_failed"} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/post-failure-key-audit.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/post-failure-key-audit.log new file mode 100644 index 00000000..a3dfd84a --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/post-failure-key-audit.log @@ -0,0 +1,3 @@ +목적: 실패 회차의 외부 키별 원장 보존 확인. 이 감사로 실패 회차를 통과로 바꾸지 않음. +EVIDENCE ledger_unique_keys=48 initial_204_keys=48 acknowledged_preserved=48 resolved_keys=48 remaining_permissions=0 +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/requests.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/requests.jsonl new file mode 100644 index 00000000..a192a8a1 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run04/requests.jsonl @@ -0,0 +1,48 @@ +{"at":"2026-10-17T06:26:56.763156+09:00","sample":0,"key":"904de47f-88c3-4179-9e02-27000600b548","retry":false,"status":204,"latency_ms":3201.244083} +{"at":"2026-10-17T06:26:56.766996+09:00","sample":1,"key":"68e944af-403d-48a6-87fb-1a3610e6c4ae","retry":false,"status":204,"latency_ms":3201.243417} +{"at":"2026-10-17T06:27:01.448324+09:00","sample":2,"key":"a7b4587e-b989-444f-b7b2-c2a0ab2c33be","retry":false,"status":204,"latency_ms":2787.333625} +{"at":"2026-10-17T06:27:01.448957+09:00","sample":3,"key":"cd593850-96b3-426c-9bd8-638cc89c9332","retry":false,"status":204,"latency_ms":2787.193875} +{"at":"2026-10-17T06:27:06.032221+09:00","sample":5,"key":"9e231ee0-3ac7-4e9e-9008-4718eb9d0a77","retry":false,"status":204,"latency_ms":2783.893583} +{"at":"2026-10-17T06:27:06.033567+09:00","sample":4,"key":"debf2d68-8cda-4101-a57c-dbb2d7a318fc","retry":false,"status":204,"latency_ms":2785.674833} +{"at":"2026-10-17T06:27:10.623256+09:00","sample":7,"key":"6032ff19-f683-4ba4-a4b8-95c82c9ca297","retry":false,"status":204,"latency_ms":2797.077959} +{"at":"2026-10-17T06:27:10.624446+09:00","sample":6,"key":"4d72e4fe-2250-4527-a8d0-dcb4bbfbefe2","retry":false,"status":204,"latency_ms":2799.237292} +{"at":"2026-10-17T06:27:15.210695+09:00","sample":9,"key":"93475ce1-9095-4d5d-b2af-1eac467572b1","retry":false,"status":204,"latency_ms":2794.526} +{"at":"2026-10-17T06:27:15.211339+09:00","sample":8,"key":"371dd26e-b2d7-4dbc-a915-33ade8d96d4d","retry":false,"status":204,"latency_ms":2794.523417} +{"at":"2026-10-17T06:27:19.808569+09:00","sample":10,"key":"2e9d2208-f42c-4640-9f1f-b5cd40b5adc8","retry":false,"status":204,"latency_ms":2800.352666} +{"at":"2026-10-17T06:27:19.808778+09:00","sample":11,"key":"aa1fb8cb-884e-42cd-901e-a43c0d8f34a1","retry":false,"status":204,"latency_ms":2792.440916} +{"at":"2026-10-17T06:27:24.412031+09:00","sample":13,"key":"2538ddf3-e611-47ac-92cc-8c40ed4a6a70","retry":false,"status":204,"latency_ms":2800.417208} +{"at":"2026-10-17T06:27:24.412681+09:00","sample":12,"key":"76530dae-aa10-49b8-94cd-812317393a48","retry":false,"status":204,"latency_ms":2801.218542} +{"at":"2026-10-17T06:27:29.002625+09:00","sample":15,"key":"6ed1661f-6a49-424d-a3e4-4266d7bcb86f","retry":false,"status":204,"latency_ms":2793.539} +{"at":"2026-10-17T06:27:29.003234+09:00","sample":14,"key":"241feb61-9d7d-4fe4-afce-b106b54f300f","retry":false,"status":204,"latency_ms":2794.0485} +{"at":"2026-10-17T06:27:33.607172+09:00","sample":17,"key":"9d52b1b4-237c-4359-9e86-3f99e2d3d09f","retry":false,"status":204,"latency_ms":2799.386083} +{"at":"2026-10-17T06:27:33.609363+09:00","sample":16,"key":"6cef6747-fdde-434c-9ae1-d5d154c5bb79","retry":false,"status":204,"latency_ms":2801.59} +{"at":"2026-10-17T06:27:38.210421+09:00","sample":18,"key":"2f8b3595-d241-40ab-bef7-5938411292a0","retry":false,"status":204,"latency_ms":2805.336167} +{"at":"2026-10-17T06:27:38.210603+09:00","sample":19,"key":"85eeeb8f-5d74-421c-87b9-8938a6c8d518","retry":false,"status":204,"latency_ms":2803.735333} +{"at":"2026-10-17T06:27:42.813375+09:00","sample":20,"key":"0bfb3da5-04d3-46fe-b29e-e6e6c710577a","retry":false,"status":204,"latency_ms":2806.029167} +{"at":"2026-10-17T06:27:42.813902+09:00","sample":21,"key":"4666fed3-c426-4dd9-a45b-044aef31713c","retry":false,"status":204,"latency_ms":2806.566333} +{"at":"2026-10-17T06:27:47.408677+09:00","sample":22,"key":"e300e206-7f5f-4e1b-9f44-d74b359a5452","retry":false,"status":204,"latency_ms":2806.622917} +{"at":"2026-10-17T06:27:47.412429+09:00","sample":23,"key":"4d8f27c3-723e-4036-a3bd-272975547507","retry":false,"status":204,"latency_ms":2806.622875} +{"at":"2026-10-17T06:27:52.015179+09:00","sample":25,"key":"18dce406-2835-480c-b1ee-9aed7eaf0834","retry":false,"status":204,"latency_ms":2814.46625} +{"at":"2026-10-17T06:27:52.020613+09:00","sample":24,"key":"0c5d4886-cab6-4e4e-b391-2ea989f76239","retry":false,"status":204,"latency_ms":2819.950334} +{"at":"2026-10-17T06:27:56.615786+09:00","sample":26,"key":"710ba01a-0115-48bd-b73d-43954fab030c","retry":false,"status":204,"latency_ms":2780.102041} +{"at":"2026-10-17T06:27:56.619043+09:00","sample":27,"key":"f3c1f4a7-0fdc-47ee-a720-5a5f3ec3dbcd","retry":false,"status":204,"latency_ms":2780.179291} +{"at":"2026-10-17T06:28:01.205539+09:00","sample":29,"key":"28a16fbe-732e-4205-8462-23d55ca50c2e","retry":false,"status":204,"latency_ms":2792.413083} +{"at":"2026-10-17T06:28:01.206288+09:00","sample":28,"key":"c836041b-ac66-4815-9561-08f1cad299eb","retry":false,"status":204,"latency_ms":2792.41325} +{"at":"2026-10-17T06:28:05.812026+09:00","sample":30,"key":"719330ab-a0af-4560-a2ce-e87b335fd1b7","retry":false,"status":204,"latency_ms":2788.588375} +{"at":"2026-10-17T06:28:05.813073+09:00","sample":31,"key":"9c132e88-ec18-4547-b295-da98284310d3","retry":false,"status":204,"latency_ms":2789.009} +{"at":"2026-10-17T06:28:10.402982+09:00","sample":33,"key":"3d94c5a3-28c8-42a7-834b-04286d356779","retry":false,"status":204,"latency_ms":2794.332333} +{"at":"2026-10-17T06:28:10.403790+09:00","sample":32,"key":"4301a907-56c5-40f1-9b95-9fee61d33884","retry":false,"status":204,"latency_ms":2794.913541} +{"at":"2026-10-17T06:28:15.008250+09:00","sample":34,"key":"f2e2fa83-a2d0-43db-84e5-5d10ecf31e0f","retry":false,"status":204,"latency_ms":2806.457041} +{"at":"2026-10-17T06:28:15.011557+09:00","sample":35,"key":"22590181-9687-4e35-87f9-6b70214abb9c","retry":false,"status":204,"latency_ms":2806.457042} +{"at":"2026-10-17T06:28:19.604363+09:00","sample":36,"key":"6379427d-9bd8-4726-87d0-69a570cf4646","retry":false,"status":204,"latency_ms":2799.966292} +{"at":"2026-10-17T06:28:19.605109+09:00","sample":37,"key":"21767b32-78b8-44bb-9081-c1e1ed35b1a5","retry":false,"status":204,"latency_ms":2799.966666} +{"at":"2026-10-17T06:28:24.210843+09:00","sample":38,"key":"b5fa73c9-a838-4f37-bc03-b23b1316d576","retry":false,"status":204,"latency_ms":2803.0105} +{"at":"2026-10-17T06:28:24.211277+09:00","sample":39,"key":"24d3b40d-934a-4ce3-9650-aa956f686389","retry":false,"status":204,"latency_ms":2803.010542} +{"at":"2026-10-17T06:28:28.805270+09:00","sample":40,"key":"9a5693e9-01b9-4c3e-8df4-5ea0badb1cf8","retry":false,"status":204,"latency_ms":2804.961375} +{"at":"2026-10-17T06:28:28.806020+09:00","sample":41,"key":"9e9f85bc-7442-4207-aec8-7c6efc37c887","retry":false,"status":204,"latency_ms":2804.957167} +{"at":"2026-10-17T06:28:33.409752+09:00","sample":43,"key":"939368c8-e22a-48a4-bec8-990b23998cd7","retry":false,"status":204,"latency_ms":2803.530583} +{"at":"2026-10-17T06:28:33.410494+09:00","sample":42,"key":"3a462534-7706-4e26-b91e-69400a43d803","retry":false,"status":204,"latency_ms":2803.530416} +{"at":"2026-10-17T06:28:38.004392+09:00","sample":45,"key":"27481d6c-4cab-4778-9b25-d762e2b66e36","retry":false,"status":204,"latency_ms":2798.282791} +{"at":"2026-10-17T06:28:38.005447+09:00","sample":44,"key":"39efeb42-ab01-4880-bb1b-4fa2c8182c3a","retry":false,"status":204,"latency_ms":2798.282916} +{"at":"2026-10-17T06:28:42.603812+09:00","sample":46,"key":"3104ce8e-376c-4f3b-b8a1-27ecc6643606","retry":false,"status":204,"latency_ms":2804.972709} +{"at":"2026-10-17T06:28:42.604595+09:00","sample":47,"key":"8b51a5e8-7290-4dff-9b21-69df25db3e1d","retry":false,"status":204,"latency_ms":2804.972917} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05-preflight.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05-preflight.log new file mode 100644 index 00000000..eca5340c --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05-preflight.log @@ -0,0 +1,13 @@ +실행 ID: designated-standby-run05-preflight +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T06:33:00.237533+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=2 +EVIDENCE replica_alias=permission-node2 sync_state=sync +EVIDENCE replica_alias=permission-node3 sync_state=async +EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names="test-standby" +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/app.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/app.log new file mode 100644 index 00000000..0f31de87 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/app.log @@ -0,0 +1,15 @@ +{"실행_ID": "designated-standby-run05-app", "목적": "실제 격리 OpenSQL·OpenProxy의 권한 회수 HA 계약", "시작_KST": "2026-10-17T06:33:05.331351+09:00", "위치": "로컬 JVM → IAP → 격리 GCP OpenSQL·OpenProxy·역할 어댑터", "명령": "./backend/gradlew -p backend permissionHaTest --console=plain", "성공_기준": "실제 커밋 경계·원장·인가 단언 전부 통과; 실패·미실행을 구분"} +> Task :compileJava UP-TO-DATE +> Task :processResources UP-TO-DATE +> Task :classes UP-TO-DATE +> Task :compileTestJava +> Task :processTestResources UP-TO-DATE +> Task :testClasses +> Task :permissionHaTest +> Task :permissionHaTest +EVIDENCE 2026-10-17T06:33:22.721172+09:00 mode=designated scenario=standby-loss phase=fixture-start samples=48 +EVIDENCE 2026-10-17T06:33:24.508259+09:00 mode=designated scenario=standby-loss phase=fixture-ready samples=48 +EVIDENCE 2026-10-17T06:35:16.565197+09:00 mode=designated scenario=standby-loss phase=summary initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2803.802209 p95_ms=2906.397625 hikari_max_active=2 hikari_max_pending=0 +OpenSqlSynchronousPolicyComparisonTest > 동일 회수 부하 중 장애가 있어도 확정 요청이 보존되고 불명 요청은 같은 키로 수렴한다 PASSED +BUILD SUCCESSFUL in 2m 11s +{"종료_KST": "2026-10-17T06:35:16.667249+09:00", "exit_code": 0} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/control.jsonl new file mode 100644 index 00000000..c019b5d2 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/control.jsonl @@ -0,0 +1,36 @@ +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:30.472655+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "bb3c73521155a64e9675a28a32c9f673f9a47e0f"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:30.472855+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:31.364199+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:32:31.303654+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:32.277386+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:32:32.221236+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:33.161874+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:32:33.097983+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:33.962521+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T21:32:33.906474+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:34.824957+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:32:34.769194+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:35.711710+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:32:35.649987+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:36.562828+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:32:36.506183+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:37.535531+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:32:37.446688+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:42.392377+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:32:42.331149+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:50.333222+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "start", "completed": true, "at_utc": "2026-10-16T21:32:50.212076+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:55.658783+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "restore-schema", "completed": true, "at_utc": "2026-10-16T21:32:55.577793+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:56.469855+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:57.314352+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:58.148805+09:00", "phase": "remote", "node_alias": "node3", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:58.978531+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:32:59.804087+09:00", "phase": "remote", "node_alias": "node3", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:00.191229+09:00", "phase": "dynamic-observed", "configuration": {"synchronous_mode": true, "synchronous_mode_strict": true, "synchronous_node_count": 1}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:00.191431+09:00", "phase": "reset-ready", "primary": "node1", "replicas": 2, "mode": "designated", "strict": true, "node_count": 1} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:05.293934+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:05.294440+09:00", "phase": "preflight", "evidence": "EVIDENCE streaming_replicas=2"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:05.294475+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node2 sync_state=sync"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:05.294503+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node3 sync_state=async"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:05.294522+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:05.294541+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=\"test-standby\""} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:32.607778+09:00", "phase": "fault-requested", "target": "node2", "scenario": "standby-loss"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:33.593646+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:33:33.538238+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:33.593822+09:00", "phase": "fault-confirmed", "target": "node2"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:42.608456+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:33:42.545626+00:00", "production_config_unchanged": true}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:43.543712+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:44.459482+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:33:46.153676+09:00", "phase": "rejoined", "target": "node2", "role": "replica"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:35:16.676848+09:00", "phase": "app-finished", "exit_code": 0} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:35:18.567216+09:00", "phase": "key-reconciliation", "evidence": "EVIDENCE ledger_unique_keys=48 initial_204_keys=48 acknowledged_preserved=48 resolved_keys=48 remaining_permissions=0"} +{"run_id": "designated-standby-run05", "at_kst": "2026-10-17T06:35:18.567662+09:00", "phase": "summary", "measurement": {"run_id": "designated-standby-run05", "mode": "designated", "scenario": "standby-loss", "initial_requests": 48, "initial_204": 48, "initial_unconfirmed": 0, "retry_requests": 0, "max_success_gap_seconds": 4.810205, "first_204_after_fault_requested_seconds": 4.512913, "app_contract": "initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2803.802209 p95_ms=2906.397625 hikari_max_active=2 hikari_max_pending=0"}} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/key-reconciliation.log b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/key-reconciliation.log new file mode 100644 index 00000000..ad1ce0aa --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/key-reconciliation.log @@ -0,0 +1,4 @@ +목적: 외부 원장의 실제 요청 키별 성공 보존·불명 해소 대조 +위치: 현재 Primary / 같은 격리 OpenProxy 경로 +EVIDENCE ledger_unique_keys=48 initial_204_keys=48 acknowledged_preserved=48 resolved_keys=48 remaining_permissions=0 +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/measurement.json b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/measurement.json new file mode 100644 index 00000000..4aba19ad --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/measurement.json @@ -0,0 +1,12 @@ +{ + "run_id": "designated-standby-run05", + "mode": "designated", + "scenario": "standby-loss", + "initial_requests": 48, + "initial_204": 48, + "initial_unconfirmed": 0, + "retry_requests": 0, + "max_success_gap_seconds": 4.810205, + "first_204_after_fault_requested_seconds": 4.512913, + "app_contract": "initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2803.802209 p95_ms=2906.397625 hikari_max_active=2 hikari_max_pending=0" +} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/requests.jsonl b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/requests.jsonl new file mode 100644 index 00000000..140f9f64 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/requests.jsonl @@ -0,0 +1,48 @@ +{"at":"2026-10-17T06:33:27.697481+09:00","sample":0,"key":"06c0cf81-7a57-45ea-b1a9-c8a98293746d","retry":false,"status":204,"latency_ms":3185.985709} +{"at":"2026-10-17T06:33:27.701353+09:00","sample":1,"key":"5287c41d-358e-40c1-a496-9988e4b0b463","retry":false,"status":204,"latency_ms":3185.986334} +{"at":"2026-10-17T06:33:32.511558+09:00","sample":2,"key":"4eb221d2-752d-4b6c-8d47-0d231cddf01e","retry":false,"status":204,"latency_ms":2906.397625} +{"at":"2026-10-17T06:33:32.512326+09:00","sample":3,"key":"d76de61a-2abb-45d1-a759-f946b56bb131","retry":false,"status":204,"latency_ms":2906.266833} +{"at":"2026-10-17T06:33:37.120639+09:00","sample":5,"key":"736eff34-305a-4957-9a60-7a137836162c","retry":false,"status":204,"latency_ms":2816.334958} +{"at":"2026-10-17T06:33:37.121799+09:00","sample":4,"key":"1e99f48a-348c-4a5f-bd5f-c3573a7d6852","retry":false,"status":204,"latency_ms":2816.385333} +{"at":"2026-10-17T06:33:41.736375+09:00","sample":6,"key":"98e61e7f-8217-4a51-a0d4-84195d68e49d","retry":false,"status":204,"latency_ms":2801.755125} +{"at":"2026-10-17T06:33:41.736808+09:00","sample":7,"key":"6749f759-a66d-414f-b255-4c7a8ce6e6f8","retry":false,"status":204,"latency_ms":2802.195375} +{"at":"2026-10-17T06:33:46.335572+09:00","sample":8,"key":"f319d3a3-72b9-4fa2-a969-3372fe340b90","retry":false,"status":204,"latency_ms":2804.034916} +{"at":"2026-10-17T06:33:46.336561+09:00","sample":9,"key":"df522b67-144f-46be-b560-d54d9ebd5d0d","retry":false,"status":204,"latency_ms":2803.910833} +{"at":"2026-10-17T06:33:50.928388+09:00","sample":10,"key":"284581c4-c372-4a95-9a6a-fb73ca494c49","retry":false,"status":204,"latency_ms":2798.826625} +{"at":"2026-10-17T06:33:50.929147+09:00","sample":11,"key":"77279795-fec3-402f-a782-3aa046e2f33a","retry":false,"status":204,"latency_ms":2798.615625} +{"at":"2026-10-17T06:33:55.532983+09:00","sample":13,"key":"67464851-4c9a-4bcb-8e7b-345158f8ecd4","retry":false,"status":204,"latency_ms":2798.179583} +{"at":"2026-10-17T06:33:55.536182+09:00","sample":12,"key":"a2e30217-1721-43db-a2f2-646ddb027898","retry":false,"status":204,"latency_ms":2798.321125} +{"at":"2026-10-17T06:34:00.110591+09:00","sample":15,"key":"844f28ee-fe21-4e25-9178-e2af1582190e","retry":false,"status":204,"latency_ms":2788.358542} +{"at":"2026-10-17T06:34:00.111103+09:00","sample":14,"key":"8830d667-4d8b-4e0b-8f2d-812ebce3fb6d","retry":false,"status":204,"latency_ms":2788.876208} +{"at":"2026-10-17T06:34:04.906642+09:00","sample":17,"key":"18125449-3ff8-4a2a-8cff-13cda067bfdf","retry":false,"status":204,"latency_ms":2799.98125} +{"at":"2026-10-17T06:34:04.909779+09:00","sample":16,"key":"4ddd2904-7086-424f-8f0d-b35ae7b282bb","retry":false,"status":204,"latency_ms":2799.9815} +{"at":"2026-10-17T06:34:09.504602+09:00","sample":19,"key":"0a7c911b-42c8-4cb6-bc8c-eb710a2965f5","retry":false,"status":204,"latency_ms":2793.324041} +{"at":"2026-10-17T06:34:09.505290+09:00","sample":18,"key":"a5add0fb-cf75-49c5-be6c-b62ba58d27cd","retry":false,"status":204,"latency_ms":2796.699959} +{"at":"2026-10-17T06:34:14.111788+09:00","sample":20,"key":"041cea42-adc0-41cc-9a76-34255aa19cc2","retry":false,"status":204,"latency_ms":2803.510792} +{"at":"2026-10-17T06:34:14.112593+09:00","sample":21,"key":"493a079b-9b67-4fe7-aa89-f4a0c3ed8645","retry":false,"status":204,"latency_ms":2803.510542} +{"at":"2026-10-17T06:34:18.711345+09:00","sample":23,"key":"3220895c-9c84-45a5-8a2f-18277e47b63e","retry":false,"status":204,"latency_ms":2805.470416} +{"at":"2026-10-17T06:34:18.712063+09:00","sample":22,"key":"451947d8-2546-4e92-9f86-ba26d37b918b","retry":false,"status":204,"latency_ms":2805.4725} +{"at":"2026-10-17T06:34:23.309166+09:00","sample":24,"key":"5c652dd2-444d-4498-858f-5aed5c2bc471","retry":false,"status":204,"latency_ms":2803.772042} +{"at":"2026-10-17T06:34:23.309783+09:00","sample":25,"key":"f595f520-3a2f-4e52-b605-bd60e11d67e4","retry":false,"status":204,"latency_ms":2803.772666} +{"at":"2026-10-17T06:34:27.914247+09:00","sample":27,"key":"6a174425-472c-4111-990f-59d20e0c9c0f","retry":false,"status":204,"latency_ms":2811.752041} +{"at":"2026-10-17T06:34:27.917496+09:00","sample":26,"key":"20a7dbc1-016a-460a-a3d2-0abff78c1ae8","retry":false,"status":204,"latency_ms":2811.75225} +{"at":"2026-10-17T06:34:32.512008+09:00","sample":28,"key":"145abd42-458e-4162-8fd7-7a72f601bd5f","retry":false,"status":204,"latency_ms":2809.383875} +{"at":"2026-10-17T06:34:32.512711+09:00","sample":29,"key":"212069c4-ca24-4b9a-b4ae-5971f21b1cb8","retry":false,"status":204,"latency_ms":2809.383625} +{"at":"2026-10-17T06:34:37.105299+09:00","sample":31,"key":"d85d74b4-939e-4271-8066-eee7d5fd8040","retry":false,"status":204,"latency_ms":2797.517167} +{"at":"2026-10-17T06:34:37.106282+09:00","sample":30,"key":"06633e82-ada7-410e-a4bc-8249ba6f0161","retry":false,"status":204,"latency_ms":2797.517333} +{"at":"2026-10-17T06:34:41.710284+09:00","sample":33,"key":"10be50e4-5654-4ac0-b390-d6c57039df6a","retry":false,"status":204,"latency_ms":2808.552209} +{"at":"2026-10-17T06:34:41.711094+09:00","sample":32,"key":"0a7797ed-3f38-40f1-b85a-eadd32aa4e68","retry":false,"status":204,"latency_ms":2808.557916} +{"at":"2026-10-17T06:34:46.311376+09:00","sample":34,"key":"d1ec2ae8-6f3e-4535-b5ca-257bf586c3f0","retry":false,"status":204,"latency_ms":2806.8975} +{"at":"2026-10-17T06:34:46.314771+09:00","sample":35,"key":"4ebbbc93-e214-47f1-ba73-843b9b5fe43b","retry":false,"status":204,"latency_ms":2806.897625} +{"at":"2026-10-17T06:34:50.893879+09:00","sample":36,"key":"241ab112-b429-4c51-a322-6c4f1f63538f","retry":false,"status":204,"latency_ms":2790.085584} +{"at":"2026-10-17T06:34:50.895011+09:00","sample":37,"key":"db326edc-33a0-49b7-aa44-c00638981232","retry":false,"status":204,"latency_ms":2791.254084} +{"at":"2026-10-17T06:34:55.522524+09:00","sample":39,"key":"faa865ff-d3ee-41d1-b6dc-672734138abb","retry":false,"status":204,"latency_ms":2822.539875} +{"at":"2026-10-17T06:34:55.523219+09:00","sample":38,"key":"2e9b4660-981c-4195-9a33-b09746fe11ee","retry":false,"status":204,"latency_ms":2822.53975} +{"at":"2026-10-17T06:35:00.107562+09:00","sample":40,"key":"2001b322-5d27-48ab-a610-f0641cd5083e","retry":false,"status":204,"latency_ms":2803.802209} +{"at":"2026-10-17T06:35:00.108333+09:00","sample":41,"key":"09f4a652-4bc3-4c31-91ca-7b5ac3a5f7fb","retry":false,"status":204,"latency_ms":2803.802375} +{"at":"2026-10-17T06:35:04.695310+09:00","sample":43,"key":"982ad4e9-54e2-4aa8-a71a-9053ab5a10a0","retry":false,"status":204,"latency_ms":2788.191209} +{"at":"2026-10-17T06:35:04.696512+09:00","sample":42,"key":"81adfd3a-ede8-4afd-8560-67928f778797","retry":false,"status":204,"latency_ms":2789.40975} +{"at":"2026-10-17T06:35:09.306175+09:00","sample":45,"key":"e87e03c8-dc01-4821-8041-9ee5b4a057fc","retry":false,"status":204,"latency_ms":2798.350125} +{"at":"2026-10-17T06:35:09.306908+09:00","sample":44,"key":"f5dcf67b-4f2e-49ba-b3ca-f51dd0caf6df","retry":false,"status":204,"latency_ms":2804.370542} +{"at":"2026-10-17T06:35:13.911507+09:00","sample":46,"key":"ef24cb0f-2a84-4532-9777-70d5221e039d","retry":false,"status":204,"latency_ms":2806.755375} +{"at":"2026-10-17T06:35:13.912217+09:00","sample":47,"key":"57b25ffa-7e6b-4c3c-914f-9ed2119269e3","retry":false,"status":204,"latency_ms":2806.755583} diff --git a/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/summary.txt b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/summary.txt new file mode 100644 index 00000000..20ffe4b6 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/designated-standby-run05/summary.txt @@ -0,0 +1 @@ +initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2803.802209 p95_ms=2906.397625 hikari_max_active=2 hikari_max_pending=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/guard-ledger-run01.jsonl b/docs/test-results/opensql-sync-quorum-20261017/guard-ledger-run01.jsonl new file mode 100644 index 00000000..adb34551 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/guard-ledger-run01.jsonl @@ -0,0 +1,2 @@ +{"at_kst": "2026-10-17T07:04:54.382255+09:00", "run_id": "guard-ledger-run01", "event": "start", "purpose": "이전 응답 원장이 있는 실행 디렉터리의 재사용 차단", "location": "로컬 Gradle 사전 조건 검사, DB 접속 없음", "command": "./backend/gradlew -p backend permissionHaTest --offline --console=plain", "criteria": "예상 사전 조건 메시지와 exit 1, 신규 응답 원장 없음 또는 이전 원장 동일"} +{"at_kst": "2026-10-17T07:04:55.049493+09:00", "run_id": "guard-ledger-run01", "event": "result", "exit_code": 1, "expected_guard_seen": true, "ledger_unchanged": true, "passed": true} diff --git a/docs/test-results/opensql-sync-quorum-20261017/guard-migration-run01.jsonl b/docs/test-results/opensql-sync-quorum-20261017/guard-migration-run01.jsonl new file mode 100644 index 00000000..bcc33d11 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/guard-migration-run01.jsonl @@ -0,0 +1,2 @@ +{"at_kst": "2026-10-17T07:04:52.508075+09:00", "run_id": "guard-migration-run01", "event": "start", "purpose": "격리 외 migration 주소의 사전 차단", "location": "로컬 Gradle 사전 조건 검사, DB 접속 없음", "command": "./backend/gradlew -p backend permissionHaTest --offline --console=plain", "criteria": "예상 사전 조건 메시지와 exit 1, 신규 응답 원장 없음 또는 이전 원장 동일"} +{"at_kst": "2026-10-17T07:04:54.381654+09:00", "run_id": "guard-migration-run01", "event": "result", "exit_code": 1, "expected_guard_seen": true, "ledger_unchanged": true, "passed": true} diff --git a/docs/test-results/opensql-sync-quorum-20261017/measurement-protocol.md b/docs/test-results/opensql-sync-quorum-20261017/measurement-protocol.md new file mode 100644 index 00000000..d5437222 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/measurement-protocol.md @@ -0,0 +1,35 @@ +# 격리 동기 정책 비교의 판정·계측 경계 + +## 사전 조건 + +- 실제 OpenSQL PostgreSQL 17.8·Patroni 4.0.5, 원본 OpenProxy와 동일 역할 호환 어댑터. +- 지정형/Quorum 모두 strict=true, synchronous_node_count=1, 쓰기 synchronous_commit=on. +- 각 회차의 데이터·DCS 시험 namespace를 초기화하고 동일 호스트를 초기 Primary로 시작한다. Standby 두 대가 streaming인지 실제 SQL로 확인한다. +- 스키마와 Flyway 이력만 동일하게 사전 적재한다. 문서 데이터가 없는 스키마 snapshot SHA-256: `6c730ecef5fbb9587e57f325f18421b2f663ae057eb42078fb29446bdc17d219`. +- 합성 사용자·문서·READ 권한 48개를 직접 Primary에서 배치로 준비한다. 준비 시간은 장애 지연에 포함하지 않는다. +- 측정 JVM·Controller·JPA·Hikari·부하는 동일하다. 동시성 2, 표본 48개, 요청마다 독립 UUID. 모델·검색 품질·공개 로그인·GCS는 포함하지 않는다. + +## 장애 + +- 첫 4개 응답 뒤 지정 동기 후보인 시험 node2를 강제 종료하고 약 8초 대기 후 시작한다. +- 별도 회차에서는 시험 Primary node1을 강제 종료해 **후보를 지정하지 않은 자동 승격**을 관찰한다. 과거 #3310의 후보 지정 수동 전환 경로와 구분한다. +- 실제 종료 완료·새 Primary 관측·재가입 시각은 외부 제어 원장에 KST로 기록한다. + +## 통과 기준 + +- 최초 204를 받은 요청 키가 모두 현재 Primary에 남아야 한다. +- 결과 불명은 같은 키 WAL 재주장으로 해소하고, 최종 요청 키 48개·권한 행 0개로 수렴해야 한다. +- 응답 뒤 최종 인가에서 접근 허용 0건. 권위 조회 실패는 허용으로 집계하지 않고 별도 건수로 보존한다. +- 최종 대조도 read-write Transaction으로 현재 Primary에 고정한다. 추가 독립 JDBC 감사로 실제 외부 요청 키들을 대조한다. + +## 시간 해석 + +- `max_initial_204_gap_seconds`: 계속 제공한 최초 요청 48개 중 연속 204 응답 사이의 최대 간격이다. 준비·의도적 재시도 대기 구간은 제외한다. +- 쓰기 재개: Standby 장애는 종료 확인 이후 첫 204, Primary 장애는 새 Primary 관측 이후 첫 204를 사용한다. 시작은 실제 종료 확인 시각이다. +- 지연 p50/p95는 최초 48개 응답 전체(불명 포함)의 MVC 호출 시간을 정렬해 nearest-rank로 계산한다. +- Hikari active/pending 최대값은 100 ms 간격 표본이다. 순간 최대를 놓칠 수 있다. +- 이 수치는 로컬 JVM→IAP→격리 OpenProxy·DB 경로이며 공개 서비스 전체 RTO·성능 SLA가 아니다. 기존 라이선스 VM의 물리 자원을 공유하므로 성능 우열을 일반화하지 않는다. + +## 원본과 호환 계층 + +원본 OpenProxy는 native sync_standby와 quorum_standby를 모두 거부했다. 성공한 비교는 역할 어댑터 경유 결과이며, 원본 공급사 공식 동기/quorum 지원으로 표현하지 않는다. diff --git a/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run01.log b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run01.log new file mode 100644 index 00000000..08c97528 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run01.log @@ -0,0 +1,10 @@ +실행 ID: preflight-jdbc-run01 +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T05:43:48.464018+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=2 +EVIDENCE alias=proxy-2 error_class=PSQLException sqlstate=08001 +exit_code=1 diff --git a/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run02.log b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run02.log new file mode 100644 index 00000000..7433b805 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run02.log @@ -0,0 +1,10 @@ +실행 ID: preflight-jdbc-run02 +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T05:50:29.446303+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=2 +EVIDENCE alias=proxy-2 error_class=PSQLException sqlstate=08001 +exit_code=1 diff --git a/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run03.log b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run03.log new file mode 100644 index 00000000..d8cc7932 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run03.log @@ -0,0 +1,10 @@ +실행 ID: preflight-jdbc-run03 +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T05:55:03.760811+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=* +EVIDENCE streaming_replicas=0 +EVIDENCE alias=direct-primary error_class=IllegalStateException sqlstate=unavailable +exit_code=1 diff --git a/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run04.log b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run04.log new file mode 100644 index 00000000..db385ab7 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run04.log @@ -0,0 +1,10 @@ +실행 ID: preflight-jdbc-run04 +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T05:57:14.061567+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names="test-standby" +EVIDENCE streaming_replicas=2 +EVIDENCE alias=proxy-2 error_class=PSQLException sqlstate=08001 +exit_code=1 diff --git a/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run05.log b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run05.log new file mode 100644 index 00000000..210f4788 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/preflight-jdbc-run05.log @@ -0,0 +1,11 @@ +실행 ID: preflight-jdbc-run05 +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T06:09:45.584896+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +EVIDENCE streaming_replicas=2 +EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01-preflight.log b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01-preflight.log new file mode 100644 index 00000000..3052f535 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01-preflight.log @@ -0,0 +1,13 @@ +실행 ID: quorum-primary-run01-preflight +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T06:43:17.687755+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +EVIDENCE streaming_replicas=2 +EVIDENCE replica_alias=permission-node2 sync_state=quorum +EVIDENCE replica_alias=permission-node3 sync_state=quorum +EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/app.log b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/app.log new file mode 100644 index 00000000..236c90a0 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/app.log @@ -0,0 +1,15 @@ +{"실행_ID": "quorum-primary-run01-app", "목적": "실제 격리 OpenSQL·OpenProxy의 권한 회수 HA 계약", "시작_KST": "2026-10-17T06:43:22.767564+09:00", "위치": "로컬 JVM → IAP → 격리 GCP OpenSQL·OpenProxy·역할 어댑터", "명령": "./backend/gradlew -p backend permissionHaTest --console=plain", "성공_기준": "실제 커밋 경계·원장·인가 단언 전부 통과; 실패·미실행을 구분"} +> Task :compileJava UP-TO-DATE +> Task :processResources UP-TO-DATE +> Task :classes UP-TO-DATE +> Task :compileTestJava UP-TO-DATE +> Task :processTestResources UP-TO-DATE +> Task :testClasses UP-TO-DATE +> Task :permissionHaTest +> Task :permissionHaTest +EVIDENCE 2026-10-17T06:43:39.559053+09:00 mode=quorum scenario=primary-loss phase=fixture-start samples=48 +EVIDENCE 2026-10-17T06:43:41.357151+09:00 mode=quorum scenario=primary-loss phase=fixture-ready samples=48 +EVIDENCE 2026-10-17T06:46:18.772061+09:00 mode=quorum scenario=primary-loss phase=summary initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2804.792959 p95_ms=5019.69925 hikari_max_active=2 hikari_max_pending=2 +OpenSqlSynchronousPolicyComparisonTest > 동일 회수 부하 중 장애가 있어도 확정 요청이 보존되고 불명 요청은 같은 키로 수렴한다 PASSED +BUILD SUCCESSFUL in 2m 56s +{"종료_KST": "2026-10-17T06:46:18.854394+09:00", "exit_code": 0} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/control.jsonl new file mode 100644 index 00000000..eaba200d --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/control.jsonl @@ -0,0 +1,36 @@ +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:47.872173+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "bb3c73521155a64e9675a28a32c9f673f9a47e0f"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:47.872641+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:48.779267+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:42:48.723592+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:49.692339+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:42:49.641445+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:50.589717+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:42:50.532709+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:51.392929+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T21:42:51.340938+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:52.253455+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:42:52.195969+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:53.142244+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:42:53.088282+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:54.030320+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:42:53.978855+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:54.994549+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:42:54.936442+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:42:59.893985+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:42:59.833299+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:07.756055+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "start", "completed": true, "at_utc": "2026-10-16T21:43:07.698584+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:13.028458+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "restore-schema", "completed": true, "at_utc": "2026-10-16T21:43:12.952889+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:13.824626+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:14.670181+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:15.544485+09:00", "phase": "remote", "node_alias": "node3", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:16.388438+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:17.251227+09:00", "phase": "remote", "node_alias": "node3", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:17.638801+09:00", "phase": "dynamic-observed", "configuration": {"synchronous_mode": "quorum", "synchronous_mode_strict": true, "synchronous_node_count": 1}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:17.638996+09:00", "phase": "reset-ready", "primary": "node1", "replicas": 2, "mode": "quorum", "strict": true, "node_count": 1} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:22.728759+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=ANY 1 (\"test-standby\",\"test-standby\")"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:22.728947+09:00", "phase": "preflight", "evidence": "EVIDENCE streaming_replicas=2"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:22.728970+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node2 sync_state=quorum"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:22.728995+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node3 sync_state=quorum"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:22.729009+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=ANY 1 (\"test-standby\",\"test-standby\")"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:22.729027+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=ANY 1 (\"test-standby\",\"test-standby\")"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:49.385366+09:00", "phase": "fault-requested", "target": "node1", "scenario": "primary-loss"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:50.324467+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:43:50.273210+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:43:50.324544+09:00", "phase": "fault-confirmed", "target": "node1"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:44:09.019871+09:00", "phase": "automatic-promotion", "new_primary": "node3"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:44:10.035850+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:44:09.972250+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:44:10.958968+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:44:15.020128+09:00", "phase": "rejoined", "target": "node1", "role": "replica"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:46:18.893599+09:00", "phase": "app-finished", "exit_code": 0} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:46:20.794722+09:00", "phase": "key-reconciliation", "evidence": "EVIDENCE ledger_unique_keys=48 initial_204_keys=39 acknowledged_preserved=39 resolved_keys=48 remaining_permissions=0"} +{"run_id": "quorum-primary-run01", "at_kst": "2026-10-17T06:46:20.795891+09:00", "phase": "summary", "measurement": {"run_id": "quorum-primary-run01", "mode": "quorum", "scenario": "primary-loss", "initial_requests": 48, "initial_204": 39, "initial_unconfirmed": 9, "retry_requests": 9, "max_success_gap_seconds": 26.969831, "first_204_after_fault_requested_seconds": 26.841306, "app_contract": "initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2804.792959 p95_ms=5019.69925 hikari_max_active=2 hikari_max_pending=2"}} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/key-reconciliation.log b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/key-reconciliation.log new file mode 100644 index 00000000..923f88bd --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/key-reconciliation.log @@ -0,0 +1,4 @@ +목적: 외부 원장의 실제 요청 키별 성공 보존·불명 해소 대조 +위치: 현재 Primary / 같은 격리 OpenProxy 경로 +EVIDENCE ledger_unique_keys=48 initial_204_keys=39 acknowledged_preserved=39 resolved_keys=48 remaining_permissions=0 +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/measurement.json b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/measurement.json new file mode 100644 index 00000000..2070921c --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/measurement.json @@ -0,0 +1,12 @@ +{ + "run_id": "quorum-primary-run01", + "mode": "quorum", + "scenario": "primary-loss", + "initial_requests": 48, + "initial_204": 39, + "initial_unconfirmed": 9, + "retry_requests": 9, + "max_success_gap_seconds": 26.969831, + "first_204_after_fault_requested_seconds": 26.841306, + "app_contract": "initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2804.792959 p95_ms=5019.69925 hikari_max_active=2 hikari_max_pending=2" +} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/requests.jsonl b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/requests.jsonl new file mode 100644 index 00000000..06705f25 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/requests.jsonl @@ -0,0 +1,57 @@ +{"at":"2026-10-17T06:43:44.555547+09:00","sample":1,"key":"11f9c1a7-53a1-49ad-be6e-f1b242369322","retry":false,"status":204,"latency_ms":3197.430541} +{"at":"2026-10-17T06:43:44.559181+09:00","sample":0,"key":"2c522d26-e980-4786-817e-0ba29ea77fe3","retry":false,"status":204,"latency_ms":3197.444125} +{"at":"2026-10-17T06:43:49.255367+09:00","sample":2,"key":"4ba33583-c5fb-464b-918e-950c3185edd7","retry":false,"status":204,"latency_ms":2796.888417} +{"at":"2026-10-17T06:43:49.256783+09:00","sample":3,"key":"84119700-df4a-4a8d-ae52-201dc383d930","retry":false,"status":204,"latency_ms":2797.444375} +{"at":"2026-10-17T06:43:50.678958+09:00","sample":4,"key":"275d43b7-763d-49dd-883b-35f483e6173b","retry":false,"status":503,"latency_ms":414.074667} +{"at":"2026-10-17T06:43:55.284596+09:00","sample":5,"key":"aa08765c-0785-4c73-943b-6b07f1e8ed7a","retry":false,"status":503,"latency_ms":5019.69925} +{"at":"2026-10-17T06:43:55.693053+09:00","sample":6,"key":"d3fcf557-12d3-42f3-9a48-8c033b7ac09f","retry":false,"status":503,"latency_ms":5013.715083} +{"at":"2026-10-17T06:44:00.298435+09:00","sample":7,"key":"aafb2bd5-8b97-4ac7-9e24-68c6198fc8b7","retry":false,"status":503,"latency_ms":5013.374959} +{"at":"2026-10-17T06:44:00.706875+09:00","sample":8,"key":"688832af-767f-48ba-839e-f8a5c131fd7b","retry":false,"status":503,"latency_ms":5013.3455} +{"at":"2026-10-17T06:44:05.305308+09:00","sample":9,"key":"cc2bae8f-cc62-4d79-9805-8bcb9c9a31b8","retry":false,"status":503,"latency_ms":5006.111125} +{"at":"2026-10-17T06:44:05.718142+09:00","sample":10,"key":"50ed7757-683e-458d-89d2-8f1e60bbb577","retry":false,"status":503,"latency_ms":5010.714792} +{"at":"2026-10-17T06:44:10.318005+09:00","sample":11,"key":"c31d8f6a-45e7-4489-b5a3-e36cf60d571e","retry":false,"status":503,"latency_ms":5012.43925} +{"at":"2026-10-17T06:44:10.733227+09:00","sample":12,"key":"67358ec2-1e53-4517-9c81-8a2c5c587ccc","retry":false,"status":503,"latency_ms":5014.357833} +{"at":"2026-10-17T06:44:16.226614+09:00","sample":13,"key":"8e732b1d-1e2c-4dd7-96a4-0980a96e3226","retry":false,"status":204,"latency_ms":5907.1955} +{"at":"2026-10-17T06:44:17.235406+09:00","sample":14,"key":"c1bf0782-4d87-4805-901c-601f2cc93c68","retry":false,"status":204,"latency_ms":6501.72925} +{"at":"2026-10-17T06:44:20.831222+09:00","sample":15,"key":"14a16878-f109-4124-8c30-024ff453c640","retry":false,"status":204,"latency_ms":2808.444208} +{"at":"2026-10-17T06:44:21.822552+09:00","sample":16,"key":"3a137d6d-55e4-4d58-ba8b-687aa7e9935e","retry":false,"status":204,"latency_ms":2793.204583} +{"at":"2026-10-17T06:44:25.423958+09:00","sample":17,"key":"2c538946-aeb6-4a0d-89b0-6b5c51c459fc","retry":false,"status":204,"latency_ms":2797.73475} +{"at":"2026-10-17T06:44:26.428702+09:00","sample":18,"key":"345ed486-9c4b-4e58-aa85-2211ee3e1cc6","retry":false,"status":204,"latency_ms":2815.950667} +{"at":"2026-10-17T06:44:30.000256+09:00","sample":19,"key":"34b52885-a0cf-4d40-8884-fc7b4889e854","retry":false,"status":204,"latency_ms":2776.282209} +{"at":"2026-10-17T06:44:30.978523+09:00","sample":20,"key":"0a40e647-5067-4b40-ba6b-8d283a766d10","retry":false,"status":204,"latency_ms":2768.309958} +{"at":"2026-10-17T06:44:34.565469+09:00","sample":21,"key":"dc941f34-3214-4f27-bd07-bd53dc56756a","retry":false,"status":204,"latency_ms":2787.63175} +{"at":"2026-10-17T06:44:35.560553+09:00","sample":22,"key":"5fdd9ec8-5b69-4bda-ac8b-264c9de62a25","retry":false,"status":204,"latency_ms":2799.811417} +{"at":"2026-10-17T06:44:39.562232+09:00","sample":23,"key":"bd16d4e3-4814-4b14-bae6-4da900c88a1c","retry":false,"status":204,"latency_ms":3201.491542} +{"at":"2026-10-17T06:44:40.556283+09:00","sample":24,"key":"0142263b-4f01-4d50-969a-0ab51b16474e","retry":false,"status":204,"latency_ms":2764.924083} +{"at":"2026-10-17T06:44:44.159313+09:00","sample":25,"key":"3193fbea-cff7-49c9-bdc8-6f772e46b510","retry":false,"status":204,"latency_ms":2780.727834} +{"at":"2026-10-17T06:44:45.159101+09:00","sample":26,"key":"13081566-0a92-432a-b697-1b6eb06dd0b6","retry":false,"status":204,"latency_ms":2795.127958} +{"at":"2026-10-17T06:44:48.745424+09:00","sample":27,"key":"0631b105-984d-4aac-94c9-8e80cbcc0da2","retry":false,"status":204,"latency_ms":2795.499709} +{"at":"2026-10-17T06:44:49.713576+09:00","sample":28,"key":"73e3d58d-4c18-4ead-b8e7-19810d707856","retry":false,"status":204,"latency_ms":2762.237458} +{"at":"2026-10-17T06:44:53.370846+09:00","sample":29,"key":"4b840dc2-442c-4158-8e5e-9c97843d52d8","retry":false,"status":204,"latency_ms":2812.881666} +{"at":"2026-10-17T06:44:54.267331+09:00","sample":30,"key":"7b9a22c2-9c33-42a2-af03-77e40c1a6d92","retry":false,"status":204,"latency_ms":2780.642041} +{"at":"2026-10-17T06:44:57.968179+09:00","sample":31,"key":"12667b15-e345-4e75-9cdf-6c801f75d749","retry":false,"status":204,"latency_ms":2804.792959} +{"at":"2026-10-17T06:44:58.859737+09:00","sample":32,"key":"c37e8c74-5e6e-4106-86e8-3cd4516393fa","retry":false,"status":204,"latency_ms":2796.587375} +{"at":"2026-10-17T06:45:02.555074+09:00","sample":33,"key":"d19f6c21-6601-49d1-9d89-d29e2294dee9","retry":false,"status":204,"latency_ms":2793.416334} +{"at":"2026-10-17T06:45:03.453424+09:00","sample":34,"key":"f14509f7-a8b1-4db6-b08e-5c859eeec2e3","retry":false,"status":204,"latency_ms":2781.442334} +{"at":"2026-10-17T06:45:07.221082+09:00","sample":35,"key":"ba1164ae-23b5-4eee-9cbb-a49a7fb107f6","retry":false,"status":204,"latency_ms":2851.993875} +{"at":"2026-10-17T06:45:08.074991+09:00","sample":36,"key":"b0f3b6ef-7ed5-4e7f-9355-c4dceab8bad9","retry":false,"status":204,"latency_ms":2815.163542} +{"at":"2026-10-17T06:45:11.761497+09:00","sample":37,"key":"060f2915-2c25-47c2-a5ea-5fb2710231db","retry":false,"status":204,"latency_ms":2770.086666} +{"at":"2026-10-17T06:45:12.665721+09:00","sample":38,"key":"2fbdc862-eff6-4dc6-8885-327983df1c1d","retry":false,"status":204,"latency_ms":2813.2405} +{"at":"2026-10-17T06:45:16.362321+09:00","sample":39,"key":"bb93dd8d-e63f-4d6e-89e9-e4c103594d4e","retry":false,"status":204,"latency_ms":2819.3575} +{"at":"2026-10-17T06:45:17.247865+09:00","sample":40,"key":"35a161ac-f10a-4943-abf9-6e66605fcf82","retry":false,"status":204,"latency_ms":2791.330791} +{"at":"2026-10-17T06:45:20.969873+09:00","sample":41,"key":"84b49ef3-10b1-4be0-a2b4-9b2f7e253f61","retry":false,"status":204,"latency_ms":2808.720541} +{"at":"2026-10-17T06:45:21.879907+09:00","sample":42,"key":"c79bc418-c968-4db7-93bd-0821e81f2a83","retry":false,"status":204,"latency_ms":2808.16175} +{"at":"2026-10-17T06:45:25.561648+09:00","sample":43,"key":"4c52c4fc-635e-4003-9834-a8e62ce9a2ee","retry":false,"status":204,"latency_ms":2799.414875} +{"at":"2026-10-17T06:45:26.461427+09:00","sample":44,"key":"3f4eb0b4-6c29-4995-b7cc-bd1f4bbae03d","retry":false,"status":204,"latency_ms":2791.452334} +{"at":"2026-10-17T06:45:30.179864+09:00","sample":45,"key":"fdca76eb-ab6c-4c3f-9ab6-6de9fa70fa95","retry":false,"status":204,"latency_ms":2823.45075} +{"at":"2026-10-17T06:45:31.066454+09:00","sample":46,"key":"e6bd5bdd-c948-495a-b9da-9ee51217210c","retry":false,"status":204,"latency_ms":2815.628667} +{"at":"2026-10-17T06:45:34.766636+09:00","sample":47,"key":"4c4ec9fe-b048-4da7-a691-5f52afbec978","retry":false,"status":204,"latency_ms":2799.494625} +{"at":"2026-10-17T06:45:39.369993+09:00","sample":4,"key":"275d43b7-763d-49dd-883b-35f483e6173b","retry":true,"status":204,"latency_ms":2762.8265} +{"at":"2026-10-17T06:45:43.964326+09:00","sample":5,"key":"aa08765c-0785-4c73-943b-6b07f1e8ed7a","retry":true,"status":204,"latency_ms":2804.159208} +{"at":"2026-10-17T06:45:48.563962+09:00","sample":6,"key":"d3fcf557-12d3-42f3-9a48-8c033b7ac09f","retry":true,"status":204,"latency_ms":2806.570583} +{"at":"2026-10-17T06:45:53.163582+09:00","sample":7,"key":"aafb2bd5-8b97-4ac7-9e24-68c6198fc8b7","retry":true,"status":204,"latency_ms":2801.821667} +{"at":"2026-10-17T06:45:57.764517+09:00","sample":8,"key":"688832af-767f-48ba-839e-f8a5c131fd7b","retry":true,"status":204,"latency_ms":2809.641042} +{"at":"2026-10-17T06:46:02.368466+09:00","sample":9,"key":"cc2bae8f-cc62-4d79-9805-8bcb9c9a31b8","retry":true,"status":204,"latency_ms":2805.547625} +{"at":"2026-10-17T06:46:06.972138+09:00","sample":10,"key":"50ed7757-683e-458d-89d2-8f1e60bbb577","retry":true,"status":204,"latency_ms":2809.525333} +{"at":"2026-10-17T06:46:11.565899+09:00","sample":11,"key":"c31d8f6a-45e7-4489-b5a3-e36cf60d571e","retry":true,"status":204,"latency_ms":2809.662} +{"at":"2026-10-17T06:46:16.158224+09:00","sample":12,"key":"67358ec2-1e53-4517-9c81-8a2c5c587ccc","retry":true,"status":204,"latency_ms":2801.449125} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/summary.txt b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/summary.txt new file mode 100644 index 00000000..7ce0888f --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-primary-run01/summary.txt @@ -0,0 +1 @@ +initial_requests=48 initial_204=39 initial_unknown=9 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=2 p50_ms=2804.792959 p95_ms=5019.69925 hikari_max_active=2 hikari_max_pending=2 diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01-preflight.log b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01-preflight.log new file mode 100644 index 00000000..2a96ae70 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01-preflight.log @@ -0,0 +1,13 @@ +실행 ID: quorum-standby-run01-preflight +목적: 실제 격리 OpenSQL·두 OpenProxy의 read-write Primary 도착·동기 적용 확인 +위치: 로컬 JDBC → IAP → 격리 GCP 컨테이너 +명령: PermissionHaPreflight (pgJDBC 42.7.11) +성공 기준: Primary 도착 3경로·streaming 2대·on·동기 후보 확인 +시작: 2026-10-17T06:36:57.368101+09:00 +EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +EVIDENCE streaming_replicas=2 +EVIDENCE replica_alias=permission-node2 sync_state=quorum +EVIDENCE replica_alias=permission-node3 sync_state=quorum +EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=ANY 1 ("test-standby","test-standby") +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/app.log b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/app.log new file mode 100644 index 00000000..9c9638a1 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/app.log @@ -0,0 +1,15 @@ +{"실행_ID": "quorum-standby-run01-app", "목적": "실제 격리 OpenSQL·OpenProxy의 권한 회수 HA 계약", "시작_KST": "2026-10-17T06:37:02.454475+09:00", "위치": "로컬 JVM → IAP → 격리 GCP OpenSQL·OpenProxy·역할 어댑터", "명령": "./backend/gradlew -p backend permissionHaTest --console=plain", "성공_기준": "실제 커밋 경계·원장·인가 단언 전부 통과; 실패·미실행을 구분"} +> Task :compileJava UP-TO-DATE +> Task :processResources UP-TO-DATE +> Task :classes UP-TO-DATE +> Task :compileTestJava UP-TO-DATE +> Task :processTestResources UP-TO-DATE +> Task :testClasses UP-TO-DATE +> Task :permissionHaTest +> Task :permissionHaTest +EVIDENCE 2026-10-17T06:37:19.078183+09:00 mode=quorum scenario=standby-loss phase=fixture-start samples=48 +EVIDENCE 2026-10-17T06:37:20.859442+09:00 mode=quorum scenario=standby-loss phase=fixture-ready samples=48 +EVIDENCE 2026-10-17T06:39:12.659421+09:00 mode=quorum scenario=standby-loss phase=summary initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2796.313417 p95_ms=2920.39875 hikari_max_active=2 hikari_max_pending=0 +OpenSqlSynchronousPolicyComparisonTest > 동일 회수 부하 중 장애가 있어도 확정 요청이 보존되고 불명 요청은 같은 키로 수렴한다 PASSED +BUILD SUCCESSFUL in 2m 10s +{"종료_KST": "2026-10-17T06:39:12.751453+09:00", "exit_code": 0} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/control.jsonl b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/control.jsonl new file mode 100644 index 00000000..1b99c563 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/control.jsonl @@ -0,0 +1,36 @@ +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:26.865936+09:00", "phase": "start", "purpose": "OpenSQL·OpenProxy 동일 MVC 회수 부하와 장애 비교", "samples": 48, "concurrency": 2, "success_criteria": "확정 쓰기 보존·동일 키 재확정·금지 내용 0건·새 Primary 추적", "revision": "bb3c73521155a64e9675a28a32c9f673f9a47e0f"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:26.866114+09:00", "phase": "reset-start", "purpose": "동일 초기 Primary·설정·데이터량으로 맞춤"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:27.753569+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:36:27.698503+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:28.689869+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:36:28.631297+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:29.592207+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:36:29.533646+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:30.406194+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-dcs", "completed": true, "at_utc": "2026-10-16T21:36:30.350222+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:31.260639+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:36:31.205326+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:32.119476+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:36:32.065842+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:32.969800+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "reset-data", "completed": true, "at_utc": "2026-10-16T21:36:32.914720+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:33.974660+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "start", "completed": true, "at_utc": "2026-10-16T21:36:33.888169+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:39.505739+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:36:39.405718+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:47.388601+09:00", "phase": "remote", "node_alias": "node3", "observation": {"node_alias": "node3", "action": "start", "completed": true, "at_utc": "2026-10-16T21:36:47.285012+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:52.717066+09:00", "phase": "remote", "node_alias": "node1", "observation": {"node_alias": "node1", "action": "restore-schema", "completed": true, "at_utc": "2026-10-16T21:36:52.630403+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:53.522767+09:00", "phase": "remote", "node_alias": "node1", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:54.356386+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:55.199827+09:00", "phase": "remote", "node_alias": "node3", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:56.067442+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:56.916277+09:00", "phase": "remote", "node_alias": "node3", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:57.318696+09:00", "phase": "dynamic-observed", "configuration": {"synchronous_mode": "quorum", "synchronous_mode_strict": true, "synchronous_node_count": 1}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:36:57.318898+09:00", "phase": "reset-ready", "primary": "node1", "replicas": 2, "mode": "quorum", "strict": true, "node_count": 1} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:02.419920+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=direct-primary recovery=false version_num=170008 commit=on sync_names=ANY 1 (\"test-standby\",\"test-standby\")"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:02.420109+09:00", "phase": "preflight", "evidence": "EVIDENCE streaming_replicas=2"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:02.420136+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node2 sync_state=quorum"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:02.420164+09:00", "phase": "preflight", "evidence": "EVIDENCE replica_alias=permission-node3 sync_state=quorum"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:02.420180+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-2 recovery=false version_num=170008 commit=on sync_names=ANY 1 (\"test-standby\",\"test-standby\")"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:02.420198+09:00", "phase": "preflight", "evidence": "EVIDENCE alias=proxy-3 recovery=false version_num=170008 commit=on sync_names=ANY 1 (\"test-standby\",\"test-standby\")"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:28.919358+09:00", "phase": "fault-requested", "target": "node2", "scenario": "standby-loss"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:29.847499+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "stop", "completed": true, "at_utc": "2026-10-16T21:37:29.795100+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:29.847726+09:00", "phase": "fault-confirmed", "target": "node2"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:38.806591+09:00", "phase": "remote", "node_alias": "node2", "observation": {"node_alias": "node2", "action": "start", "completed": true, "at_utc": "2026-10-16T21:37:38.751547+00:00", "production_config_unchanged": true}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:39.732215+09:00", "phase": "remote", "node_alias": "node2", "observation": {"adapter_launch_exit": 0, "role_only_normalization": true, "read_only": true, "scope_guard": true, "port": 18010}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:40.640467+09:00", "phase": "remote", "node_alias": "node2", "observation": {"isolated_proxy_launch_exit": 0, "proxy_listen_loopback": true, "backend_count": 3, "patroni_port": 18010, "public_proxy_changed": false}} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:37:42.318330+09:00", "phase": "rejoined", "target": "node2", "role": "replica"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:39:12.754623+09:00", "phase": "app-finished", "exit_code": 0} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:39:14.644506+09:00", "phase": "key-reconciliation", "evidence": "EVIDENCE ledger_unique_keys=48 initial_204_keys=48 acknowledged_preserved=48 resolved_keys=48 remaining_permissions=0"} +{"run_id": "quorum-standby-run01", "at_kst": "2026-10-17T06:39:14.645574+09:00", "phase": "summary", "measurement": {"run_id": "quorum-standby-run01", "mode": "quorum", "scenario": "standby-loss", "initial_requests": 48, "initial_204": 48, "initial_unconfirmed": 0, "retry_requests": 0, "max_success_gap_seconds": 4.743453, "first_204_after_fault_requested_seconds": 4.402856, "app_contract": "initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2796.313417 p95_ms=2920.39875 hikari_max_active=2 hikari_max_pending=0"}} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/key-reconciliation.log b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/key-reconciliation.log new file mode 100644 index 00000000..ad1ce0aa --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/key-reconciliation.log @@ -0,0 +1,4 @@ +목적: 외부 원장의 실제 요청 키별 성공 보존·불명 해소 대조 +위치: 현재 Primary / 같은 격리 OpenProxy 경로 +EVIDENCE ledger_unique_keys=48 initial_204_keys=48 acknowledged_preserved=48 resolved_keys=48 remaining_permissions=0 +exit_code=0 diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/measurement.json b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/measurement.json new file mode 100644 index 00000000..4129a697 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/measurement.json @@ -0,0 +1,12 @@ +{ + "run_id": "quorum-standby-run01", + "mode": "quorum", + "scenario": "standby-loss", + "initial_requests": 48, + "initial_204": 48, + "initial_unconfirmed": 0, + "retry_requests": 0, + "max_success_gap_seconds": 4.743453, + "first_204_after_fault_requested_seconds": 4.402856, + "app_contract": "initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2796.313417 p95_ms=2920.39875 hikari_max_active=2 hikari_max_pending=0" +} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/requests.jsonl b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/requests.jsonl new file mode 100644 index 00000000..c215792c --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/requests.jsonl @@ -0,0 +1,48 @@ +{"at":"2026-10-17T06:37:24.061061+09:00","sample":1,"key":"746ed4ae-441f-4e30-a03e-3f0354863400","retry":false,"status":204,"latency_ms":3198.69375} +{"at":"2026-10-17T06:37:24.064722+09:00","sample":0,"key":"9e570e44-548c-43cc-99e2-95bd1d5c0489","retry":false,"status":204,"latency_ms":3198.692625} +{"at":"2026-10-17T06:37:28.753415+09:00","sample":2,"key":"8a4517b2-e366-4945-ac60-e22cba2325eb","retry":false,"status":204,"latency_ms":2788.455916} +{"at":"2026-10-17T06:37:28.754067+09:00","sample":3,"key":"e648eea1-01bb-4855-a44b-56aa0042257e","retry":false,"status":204,"latency_ms":2788.419} +{"at":"2026-10-17T06:37:33.322146+09:00","sample":5,"key":"ca56355a-dea1-494b-b0b8-20ab53d7a5f6","retry":false,"status":204,"latency_ms":2781.515458} +{"at":"2026-10-17T06:37:33.323423+09:00","sample":4,"key":"01e8885f-26d5-4140-a04b-41d86f55c28d","retry":false,"status":204,"latency_ms":2784.704167} +{"at":"2026-10-17T06:37:37.909450+09:00","sample":7,"key":"d8ac39bc-3016-4422-83b0-f7b6ca886df2","retry":false,"status":204,"latency_ms":2791.098792} +{"at":"2026-10-17T06:37:37.910156+09:00","sample":6,"key":"209928bd-bfe9-44de-9ec5-60722c7eb802","retry":false,"status":204,"latency_ms":2791.199958} +{"at":"2026-10-17T06:37:42.507495+09:00","sample":8,"key":"f83ccf2e-e2fc-4b6e-8b0c-ff7d1181287c","retry":false,"status":204,"latency_ms":2802.889958} +{"at":"2026-10-17T06:37:42.508138+09:00","sample":9,"key":"5f155051-6ee9-4c3f-9ec0-d57b9e21bb85","retry":false,"status":204,"latency_ms":2802.881083} +{"at":"2026-10-17T06:37:47.108712+09:00","sample":10,"key":"c3d9f14e-cd17-4909-aec1-78e956c3cac8","retry":false,"status":204,"latency_ms":2795.50825} +{"at":"2026-10-17T06:37:47.111985+09:00","sample":11,"key":"4cbaa0a3-08d4-4e8f-9798-b1bcd21bb32a","retry":false,"status":204,"latency_ms":2795.495417} +{"at":"2026-10-17T06:37:51.714207+09:00","sample":12,"key":"41703392-6b7c-4429-87c1-7920de1f7239","retry":false,"status":204,"latency_ms":2793.639375} +{"at":"2026-10-17T06:37:51.715407+09:00","sample":13,"key":"625c4498-540d-46a5-8a39-7c99b5d5f741","retry":false,"status":204,"latency_ms":2794.873041} +{"at":"2026-10-17T06:37:56.306273+09:00","sample":15,"key":"f030e9f3-624a-4f36-8911-74f5650b117c","retry":false,"status":204,"latency_ms":2799.334542} +{"at":"2026-10-17T06:37:56.307034+09:00","sample":14,"key":"e028384e-46bd-4083-a18e-d567727fe254","retry":false,"status":204,"latency_ms":2799.334375} +{"at":"2026-10-17T06:38:00.908595+09:00","sample":16,"key":"64a73efc-4b85-45b6-be9e-f89f8cb165f4","retry":false,"status":204,"latency_ms":2805.009708} +{"at":"2026-10-17T06:38:00.909635+09:00","sample":17,"key":"f66de574-6835-45d0-829d-9af45a44f295","retry":false,"status":204,"latency_ms":2802.29525} +{"at":"2026-10-17T06:38:05.513140+09:00","sample":19,"key":"b537e2b6-dc8e-46a6-a856-c5305d7f3010","retry":false,"status":204,"latency_ms":2816.159625} +{"at":"2026-10-17T06:38:05.513859+09:00","sample":18,"key":"54e78c17-dc61-4206-be97-d188d5b03a12","retry":false,"status":204,"latency_ms":2816.163333} +{"at":"2026-10-17T06:38:10.108690+09:00","sample":21,"key":"4df28d34-9dbb-4ed0-94fd-dbc82cee6fb8","retry":false,"status":204,"latency_ms":2802.24875} +{"at":"2026-10-17T06:38:10.109447+09:00","sample":20,"key":"89a9dbbe-554f-4c4e-927e-fdebda3d3dc0","retry":false,"status":204,"latency_ms":2802.249042} +{"at":"2026-10-17T06:38:14.705082+09:00","sample":23,"key":"b9b3f6f6-ade1-4c59-a57d-95fbcb15f74c","retry":false,"status":204,"latency_ms":2802.573208} +{"at":"2026-10-17T06:38:14.705677+09:00","sample":22,"key":"59214e95-4a6c-4913-a770-430996024fa4","retry":false,"status":204,"latency_ms":2803.38125} +{"at":"2026-10-17T06:38:19.314667+09:00","sample":24,"key":"1f376a6d-cd49-40c7-a032-622d994c952b","retry":false,"status":204,"latency_ms":2808.855} +{"at":"2026-10-17T06:38:19.315346+09:00","sample":25,"key":"c44ed9f2-238c-4c3c-a1e1-fed919620667","retry":false,"status":204,"latency_ms":2806.7035} +{"at":"2026-10-17T06:38:23.939146+09:00","sample":27,"key":"8080f27c-2b69-4f0c-a7f9-0237ad1ab35f","retry":false,"status":204,"latency_ms":2779.645791} +{"at":"2026-10-17T06:38:23.943781+09:00","sample":26,"key":"3c4eba74-a79f-4da3-a2ce-9a18152a5837","retry":false,"status":204,"latency_ms":2784.303208} +{"at":"2026-10-17T06:38:28.531677+09:00","sample":28,"key":"99a38bfd-d3d3-40ee-8253-877251fdea97","retry":false,"status":204,"latency_ms":2794.625708} +{"at":"2026-10-17T06:38:28.532530+09:00","sample":29,"key":"7394de1a-c90e-4c5f-bf20-e04d4d760969","retry":false,"status":204,"latency_ms":2794.625792} +{"at":"2026-10-17T06:38:33.108371+09:00","sample":30,"key":"5e2d26ab-1835-4557-bcbb-ea20295b97fe","retry":false,"status":204,"latency_ms":2800.064667} +{"at":"2026-10-17T06:38:33.108985+09:00","sample":31,"key":"fa5b5f83-0ffc-437f-ad18-5a9d92fe2a3d","retry":false,"status":204,"latency_ms":2800.064167} +{"at":"2026-10-17T06:38:37.704476+09:00","sample":32,"key":"acd1bd4f-e8ce-446c-ad4e-294025b53b73","retry":false,"status":204,"latency_ms":2796.313417} +{"at":"2026-10-17T06:38:37.705178+09:00","sample":33,"key":"aee204a3-8ef8-4ea1-8d34-de90abb38197","retry":false,"status":204,"latency_ms":2795.756791} +{"at":"2026-10-17T06:38:42.302141+09:00","sample":35,"key":"3c806f0f-f23e-47c6-9324-c955d74e6816","retry":false,"status":204,"latency_ms":2796.138959} +{"at":"2026-10-17T06:38:42.302856+09:00","sample":34,"key":"0ffb9f24-4d10-477b-9208-5d8183ebf1be","retry":false,"status":204,"latency_ms":2796.154625} +{"at":"2026-10-17T06:38:46.910395+09:00","sample":37,"key":"9b74b6c1-70bc-4a83-b567-81b2f8db3e4a","retry":false,"status":204,"latency_ms":2807.390125} +{"at":"2026-10-17T06:38:46.911056+09:00","sample":36,"key":"cf3dac7b-6246-4c26-ac70-2e2b692731e4","retry":false,"status":204,"latency_ms":2807.606333} +{"at":"2026-10-17T06:38:51.488339+09:00","sample":38,"key":"6dbdc838-d325-4629-8c29-cdaf10b6aa86","retry":false,"status":204,"latency_ms":2778.927625} +{"at":"2026-10-17T06:38:51.489122+09:00","sample":39,"key":"86f61b9c-ee59-475b-baa9-f4ca49fc271a","retry":false,"status":204,"latency_ms":2778.934166} +{"at":"2026-10-17T06:38:56.232575+09:00","sample":41,"key":"36710bd2-826a-4e16-a184-eb6e1fb10ec3","retry":false,"status":204,"latency_ms":2920.39875} +{"at":"2026-10-17T06:38:56.233395+09:00","sample":40,"key":"5be1dce6-93eb-4a53-bca9-5de693987693","retry":false,"status":204,"latency_ms":2920.398542} +{"at":"2026-10-17T06:39:00.806064+09:00","sample":43,"key":"340eda48-ad10-4dbf-8b92-cb19c745ebfe","retry":false,"status":204,"latency_ms":2789.258667} +{"at":"2026-10-17T06:39:00.806326+09:00","sample":42,"key":"469d7817-4885-4c0a-8ca8-6cf794bbe63c","retry":false,"status":204,"latency_ms":2789.655542} +{"at":"2026-10-17T06:39:05.408893+09:00","sample":44,"key":"bcabe40e-010e-4b7a-a168-5e36779009c5","retry":false,"status":204,"latency_ms":2807.91075} +{"at":"2026-10-17T06:39:05.409653+09:00","sample":45,"key":"a49b79f1-ed69-4291-b9d6-a10a57146ba4","retry":false,"status":204,"latency_ms":2807.910667} +{"at":"2026-10-17T06:39:10.011224+09:00","sample":46,"key":"065b40ef-a9b3-4f45-a845-b1b6bc935e25","retry":false,"status":204,"latency_ms":2794.371792} +{"at":"2026-10-17T06:39:10.011954+09:00","sample":47,"key":"69bc3d98-ebc3-4685-939d-bf92bc2dfa4a","retry":false,"status":204,"latency_ms":2794.386125} diff --git a/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/summary.txt b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/summary.txt new file mode 100644 index 00000000..e5425fd3 --- /dev/null +++ b/docs/test-results/opensql-sync-quorum-20261017/quorum-standby-run01/summary.txt @@ -0,0 +1 @@ +initial_requests=48 initial_204=48 initial_unknown=0 final_receipts=48 final_permissions=0 forbidden_content=0 auth_unavailable=0 p50_ms=2796.313417 p95_ms=2920.39875 hikari_max_active=2 hikari_max_pending=0 diff --git a/scripts/opensql/PermissionLedgerReconcile.java b/scripts/opensql/PermissionLedgerReconcile.java new file mode 100644 index 00000000..4e283904 --- /dev/null +++ b/scripts/opensql/PermissionLedgerReconcile.java @@ -0,0 +1,36 @@ +import java.nio.file.*; +import java.sql.*; +import java.util.*; +import java.util.regex.*; + +/** 격리 HA 외부 요청 키를 현재 Primary 원장과 대조하며 DB 변경·비밀 출력은 수행하지 않는다. */ +class PermissionLedgerReconcile { + public static void main(String[] args) throws Exception { + Set all=new LinkedHashSet<>();Set acknowledged=new LinkedHashSet<>();int initial=0; + Pattern key=Pattern.compile("\"key\":\"([a-f0-9-]{36})\""); + Pattern status=Pattern.compile("\"status\":([0-9]+)"); + for(String line:Files.readAllLines(Path.of(System.getenv("PERMISSION_HA_LEDGER")))) { + if(!line.contains("\"retry\":false"))continue; + Matcher k=key.matcher(line);Matcher s=status.matcher(line); + if(!k.find()||!s.find())throw new IllegalStateException("Malformed external ledger"); + UUID id=UUID.fromString(k.group(1));all.add(id);initial++; + if(Integer.parseInt(s.group(1))==204)acknowledged.add(id); + } + if(initial!=48||all.size()!=48)throw new IllegalStateException("Expected 48 unique initial requests"); + try(Connection c=DriverManager.getConnection("jdbc:postgresql://127.0.0.1:18443/permission_test?connectTimeout=3&socketTimeout=8","postgres",System.getenv("PERMISSION_HA_PASSWORD"))) { + c.setAutoCommit(false); + try(var s=c.createStatement();var r=s.executeQuery("SELECT NOT pg_is_in_recovery() AND current_database()='postgres'")) { + r.next();if(!r.getBoolean(1))throw new IllegalStateException("Isolated Primary required"); + } + try(var s=c.prepareStatement("SELECT count(*), count(*) FILTER (WHERE r.request_id=ANY(?)), count(p.id) FROM permission_revocation_requests r LEFT JOIN document_permissions p ON p.id=r.permission_id WHERE r.request_id=ANY(?)")) { + s.setArray(1,c.createArrayOf("uuid",acknowledged.toArray()));s.setArray(2,c.createArrayOf("uuid",all.toArray())); + try(var r=s.executeQuery()) { + r.next();long total=r.getLong(1),preserved=r.getLong(2),remaining=r.getLong(3); + System.out.println("EVIDENCE ledger_unique_keys="+all.size()+" initial_204_keys="+acknowledged.size()+" acknowledged_preserved="+preserved+" resolved_keys="+total+" remaining_permissions="+remaining); + if(total!=48||preserved!=acknowledged.size()||remaining!=0)throw new IllegalStateException("Request-key reconciliation failed"); + } + } + c.commit(); + } + } +}