From 1a9c112ed2675731f33045f00e18096122e96034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sat, 17 Oct 2026 18:57:43 +0900 Subject: [PATCH 1/4] =?UTF-8?q?test:=20=EA=B6=8C=ED=95=9C=20=ED=95=84?= =?UTF-8?q?=ED=84=B0=20=EA=B2=80=EC=83=89=20=EB=88=84=EB=9D=BD=20=EC=8B=9C?= =?UTF-8?q?=ED=97=98=20=ED=95=98=EB=84=A4=EC=8A=A4=20=EC=B6=94=EA=B0=80=20?= =?UTF-8?q?(#545)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 운영 VectorSearchQueryService·HybridSearchQueryService를 그대로 호출하고 Repository만 반환 행을 기록하는 Proxy로 감싸, SQL 후보 수와 앱 필터 뒤 최종 건수를 따로 잰다. Exact·현재 계획(Custom/Generic Plan)·HNSW 강제 (ef_search·iterative_scan) 8가지 방식을 같은 Transaction의 SET LOCAL로 바꾼다. - 실제 문서를 운영 Parser·Chunker·Embedding 경로로 처리해 캐시 - 실제 Embedding 방향을 섞는 결정적 합성 확장과 허용 문서 선택 - vector.search.performance.permission.phase 속성이 있을 때만 실행 --- .../benchmark/PermissionRecallBenchmark.java | 1114 +++++++++++++++++ .../benchmark/PermissionRecallCorpus.java | 269 ++++ .../benchmark/PermissionRecallDataset.java | 217 ++++ .../benchmark/PermissionRecallMetrics.java | 45 + 4 files changed, 1645 insertions(+) create mode 100644 backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallBenchmark.java create mode 100644 backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallCorpus.java create mode 100644 backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDataset.java create mode 100644 backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallMetrics.java diff --git a/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallBenchmark.java b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallBenchmark.java new file mode 100644 index 00000000..3fec8555 --- /dev/null +++ b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallBenchmark.java @@ -0,0 +1,1114 @@ +package com.opensource.docgrid.domain.search.benchmark; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; +import javax.sql.DataSource; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.postgresql.PGConnection; +import org.postgresql.copy.CopyIn; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.jpa.repository.Query; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opensource.docgrid.domain.document.service.DocumentParserRegistry; +import com.opensource.docgrid.domain.document.service.FixedSizeChunker; +import com.opensource.docgrid.domain.embedding.client.EmbeddingClient; +import com.opensource.docgrid.domain.embedding.service.AdaptiveEmbeddingBatchPlanner; +import com.opensource.docgrid.domain.embedding.service.EmbeddingVectorSupport; +import com.opensource.docgrid.domain.search.benchmark.PermissionRecallCorpus.RealChunk; +import com.opensource.docgrid.domain.search.benchmark.PermissionRecallDataset.DocumentSpec; +import com.opensource.docgrid.domain.search.config.HybridSearchProperties; +import com.opensource.docgrid.domain.search.config.VectorSearchProperties; +import com.opensource.docgrid.domain.search.dto.VectorSearchCandidate; +import com.opensource.docgrid.domain.search.repository.VectorSearchRepository; +import com.opensource.docgrid.domain.search.repository.VectorSearchRow; +import com.opensource.docgrid.domain.search.service.query.HybridSearchQueryService; +import com.opensource.docgrid.domain.search.service.query.VectorSearchQueryService; + +import jakarta.persistence.EntityManager; +import lombok.extern.slf4j.Slf4j; + +/** + * 권한 필터에 따른 벡터 검색 누락 재현과 인덱스 방식 비교 시험(#545). + * + *

운영 {@link VectorSearchQueryService}·{@link HybridSearchQueryService}를 그대로 호출하고, 그 아래 Repository만 + * 반환 행을 기록하는 Proxy로 감싸 "SQL이 돌려준 후보 수"와 "앱 필터 뒤 최종 건수"를 따로 잰다. 방식은 서비스 호출과 같은 + * Transaction 안에서 바꾼다. Exact는 HNSW Index를 임시로 지운 뒤 Rollback하고, 나머지는 {@code SET LOCAL}로 + * Plan Cache 모드·{@code hnsw.*}·(HNSW 강제 방식만) {@code enable_sort}를 바꾼다. 방식 목록은 {@link Method}에 있다. + * Exact 결과를 정답으로 Recall을 계산한다. 운영 코드·설정·마이그레이션은 바꾸지 않는다. + * + *

{@code vector.search.performance.permission.phase} 시스템 속성이 있을 때만 실행한다. {@code prepare}는 실제 문서와 + * 질문 Embedding 캐시만 만들고, {@code measure}는 캐시로 데이터를 적재해 측정한다. 격리 스키마 + * {@value #TEST_SCHEMA}만 쓰며, 접속 대상 DB는 {@code DB_HOST}·{@code DB_PORT} 등 환경변수로 반드시 격리 DB를 지정한다. + */ +@Slf4j +@Tag("integration") +@Tag("vector-search-performance") +@ActiveProfiles("test") +@SpringBootTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@DisplayName("권한 필터 × 벡터 검색 누락·인덱스 방식 비교 시험 (#545)") +class PermissionRecallBenchmark { + + static final String PROPERTY_PREFIX = "vector.search.performance.permission."; + private static final String TEST_SCHEMA = "docgrid_permission_recall_545"; + private static final int VECTOR_DIMENSION = 1024; + private static final String FALLBACK_HNSW_INDEX = "embeddings_vector_idx"; + private static final String TOP_K_SQL = topKSql(); + + @Autowired private JdbcTemplate jdbcTemplate; + @Autowired private NamedParameterJdbcTemplate namedParameterJdbcTemplate; + @Autowired private DataSource dataSource; + @Autowired private PlatformTransactionManager transactionManager; + @Autowired private EntityManager entityManager; + @Autowired private ObjectMapper objectMapper; + @Autowired private VectorSearchRepository vectorSearchRepository; + @Autowired private VectorSearchProperties vectorSearchProperties; + @Autowired private HybridSearchProperties hybridSearchProperties; + @Autowired private DocumentParserRegistry documentParserRegistry; + @Autowired private FixedSizeChunker fixedSizeChunker; + @Autowired private AdaptiveEmbeddingBatchPlanner adaptiveEmbeddingBatchPlanner; + @Autowired private EmbeddingClient embeddingClient; + + /** 진행 중인 방식 Transaction의 방식. Repository Proxy가 HNSW 강제 여부를 판단할 때 쓴다(단일 Thread 실행). */ + private Method currentMethod; + + @DynamicPropertySource + static void configureEnvironment(DynamicPropertyRegistry registry) { + registry.add("TEST_DB_SCHEMA", () -> TEST_SCHEMA); + registry.add("jwt.secret", () -> "docgrid-permission-recall-benchmark-secret-key-2026"); + registry.add( + "spring.datasource.hikari.data-source-properties.ApplicationName", + () -> "docgrid-permission-recall-benchmark" + ); + } + + @AfterAll + void dropIsolatedSchema() { + if (Boolean.parseBoolean(System.getProperty(PROPERTY_PREFIX + "drop-schema", "true"))) { + jdbcTemplate.execute("DROP SCHEMA IF EXISTS " + TEST_SCHEMA + " CASCADE"); + } + } + + @Test + @Timeout(7_200) + @EnabledIfSystemProperty(named = PROPERTY_PREFIX + "phase", matches = "prepare") + @DisplayName("실제 문서와 질문을 운영 처리 경로로 Embedding해 캐시한다") + void prepareEmbeddingCaches() throws Exception { + Configuration configuration = Configuration.fromSystemProperties(); + PermissionRecallCorpus corpus = loadCorpus(configuration); + log.info("[PERMISSION-RECALL] corpus documents={} chunks={} skipped={} embeddingSeconds={}", + corpus.documents().size(), corpus.chunkCount(), corpus.skippedSources(), corpus.embeddingSeconds()); + if (Files.exists(configuration.questionsPath())) { + Map vectors = loadQuestionVectors(configuration, loadQuestions(configuration)); + log.info("[PERMISSION-RECALL] question vectors={}", vectors.size()); + } + assertThat(corpus.chunkCount()).isPositive(); + } + + @Test + @Timeout(14_400) + @EnabledIfSystemProperty(named = PROPERTY_PREFIX + "phase", matches = "measure") + @DisplayName("허용 문서 비율 × topK × 방식별로 SQL 후보 수·최종 건수·Recall·지연·실행 계획을 잰다") + void measurePermissionRecall() throws Exception { + Configuration configuration = Configuration.fromSystemProperties(); + assertThat(configuration.methods().get(0)).isEqualTo(Method.EXACT); + + PermissionRecallCorpus corpus = loadCorpus(configuration); + List questions = loadQuestions(configuration); + Map questionVectors = loadQuestionVectors(configuration, questions); + Long modelId = jdbcTemplate.queryForObject( + "SELECT id FROM embedding_models WHERE is_active = TRUE AND is_searchable = TRUE", Long.class); + + HnswIndex hnswIndex = currentHnswIndex(); + Map settings = verifySessionSettings(); + + Recorder recorder = new Recorder(); + VectorSearchRepository countingRepository = countingRepository(recorder); + VectorSearchQueryService vectorService = + new VectorSearchQueryService(countingRepository, vectorSearchProperties); + HybridSearchQueryService hybridService = + new HybridSearchQueryService(countingRepository, vectorSearchProperties, hybridSearchProperties); + + // 1. 합성 Chunk와 원본의 유사도는 실제 Chunk의 "다른 문서 최근접 유사도" 분포 P50~P90 안에서 고른다. + double[] nearest = PermissionRecallDataset.crossDocumentNearestSimilarities(corpus.documents()); + Distribution nearestSimilarity = Distribution.of(nearest); + double similarityLow = nearestSimilarity.p50(); + double similarityHigh = nearestSimilarity.p90(); + + List scales = new ArrayList<>(); + List rows = new ArrayList<>(); + List planSamples = new ArrayList<>(); + for (String scale : configuration.scales()) { + int targetChunks = "real".equals(scale) ? 0 : Integer.parseInt(scale); + PermissionRecallDataset dataset = PermissionRecallDataset.expand( + corpus.documents(), targetChunks, similarityLow, similarityHigh, configuration.seed()); + LoadResult load = loadDataset(dataset, modelId, hnswIndex); + log.info("[PERMISSION-RECALL] scale={} documents={} chunks={} indexBuildSeconds={}", + scale, dataset.documents().size(), dataset.chunkCount(), load.indexBuildSeconds()); + + List ratioInfos = new ArrayList<>(); + for (double ratio : configuration.ratios()) { + List permittedIds = PermissionRecallDataset + .permittedDocumentIndexes(dataset.documents().size(), ratio, configuration.seed()) + .stream().map(index -> index + 1L).sorted().toList(); + int permittedChunks = permittedIds.stream() + .mapToInt(id -> dataset.source(dataset.documents().get((int) (id - 1))).chunks().size()) + .sum(); + ratioInfos.add(new RatioInfo(ratio, permittedIds.size(), permittedChunks)); + + for (int topK : configuration.topKs()) { + Map exact = new LinkedHashMap<>(); + for (Method method : configuration.methods()) { + for (int questionIndex = 0; questionIndex < questions.size(); questionIndex++) { + Question question = questions.get(questionIndex); + QuestionOutcome outcome = measureQuestion( + configuration, method, hnswIndex, question, questionVectors.get(question.text()), + modelId, permittedIds, topK, vectorService, hybridService, recorder); + if (method == Method.EXACT) { + exact.put(question.id(), outcome); + } + if (questionIndex == 0) { + planSamples.add(new PlanSample(scale, ratio, topK, method.label(), question.id(), + outcome.rawPlan())); + } + rows.add(MeasurementRow.of(scale, ratio, permittedIds.size(), permittedChunks, topK, + topK * vectorSearchProperties.getCandidatePoolMultiplier(), method, question, + outcome, exact.get(question.id()))); + } + log.info("[PERMISSION-RECALL] scale={} ratio={} topK={} method={} done", + scale, ratio, topK, method.label()); + } + } + } + scales.add(new ScaleResult( + scale, dataset.documents().size(), corpus.documents().size(), + dataset.documents().size() - corpus.documents().size(), dataset.chunkCount(), + dataset.realChunkCount(), 1.0 - (double) dataset.realChunkCount() / dataset.chunkCount(), + similarityLow, similarityHigh, load.loadSeconds(), load.indexBuildSeconds(), + load.tableBytes(), load.indexBytes(), load.totalBytes(), List.copyOf(ratioInfos))); + } + + Report report = new Report( + Instant.now().toString(), + jdbcTemplate.queryForObject("SHOW server_version", String.class), + jdbcTemplate.queryForObject("SELECT extversion FROM pg_extension WHERE extname = 'vector'", String.class), + settings, + System.getProperty("os.arch"), + System.getProperty("java.version"), + jdbcDriverVersion(), + corpus.embeddingModel(), + corpus.documents().size(), + corpus.chunkCount(), + corpus.skippedSources(), + corpus.embeddingSeconds(), + nearestSimilarity, + new SearchSettings( + vectorSearchProperties.getMinSimilarity().doubleValue(), + vectorSearchProperties.getMaxChunksPerDocument(), + vectorSearchProperties.getCandidatePoolMultiplier(), + hybridSearchProperties.isEnabled(), + hybridSearchProperties.getVectorMinSimilarity().doubleValue(), + hybridSearchProperties.getMinCoverage().doubleValue(), + hybridSearchProperties.getCoverageWeight().doubleValue(), + hybridSearchProperties.getLexicalCandidatePoolMultiplier()), + hnswIndex.definition(), + configuration, + List.copyOf(scales), + CombinationSummary.summarize(rows), + List.copyOf(rows) + ); + writeJson(configuration.outputPath(), report); + writeJson(configuration.planOutputPath(), planSamples); + assertThat(rows).isNotEmpty(); + } + + private QuestionOutcome measureQuestion( + Configuration configuration, + Method method, + HnswIndex hnswIndex, + Question question, + float[] vector, + Long modelId, + List permittedIds, + int topK, + VectorSearchQueryService vectorService, + HybridSearchQueryService hybridService, + Recorder recorder + ) throws IOException { + int candidateLimit = topK * vectorSearchProperties.getCandidatePoolMultiplier(); + + // 1. 같은 방식 설정으로 운영 findTopK SQL의 실행 계획을 한 번 남긴다(EXPLAIN ANALYZE는 사용자 지연과 별도). + String rawPlan = inMethodTransaction(method, hnswIndex, () -> { + setSortForTopK(method, false); + return namedParameterJdbcTemplate.queryForObject( + "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + TOP_K_SQL, + new MapSqlParameterSource() + .addValue("queryVector", vectorString(vector)) + .addValue("modelId", modelId) + .addValue("permittedIds", permittedIds) + .addValue("topK", candidateLimit), + String.class); + }); + PlanSummary plan = PlanSummary.parse(objectMapper.readTree(rawPlan), hnswIndex.name()); + + // 2. 예열 후 벡터 단독·하이브리드 경로를 같은 방식 설정으로 번갈아 호출하고, 첫 측정 결과를 기준으로 쓴다. + // Generic Plan 방식은 한 번이 수 초라 예열·반복을 별도 설정으로 줄인다. + boolean generic = method.planCacheMode() == PlanCacheMode.GENERIC; + int warmUpRuns = generic ? configuration.genericWarmUpRuns() : configuration.warmUpRuns(); + int measuredRuns = generic ? configuration.genericMeasuredRuns() : configuration.measuredRuns(); + for (int run = 0; run < warmUpRuns; run++) { + inMethodTransaction(method, hnswIndex, () -> vectorService.search(vector, modelId, permittedIds, topK)); + inMethodTransaction(method, hnswIndex, + () -> hybridService.search(vector, question.text(), modelId, permittedIds, topK)); + } + + List vectorMillis = new ArrayList<>(); + List hybridMillis = new ArrayList<>(); + List sqlChunkIds = null; + List vectorFinalIds = null; + List hybridFinalIds = null; + int hybridLexicalRows = 0; + boolean stable = true; + for (int run = 0; run < measuredRuns; run++) { + recorder.reset(); + Timed> vectorResult = inMethodTransaction(method, hnswIndex, + () -> timed(() -> vectorService.search(vector, modelId, permittedIds, topK))); + List runSqlIds = recorder.topKChunkIds(); + List runVectorIds = chunkIds(vectorResult.value()); + + recorder.reset(); + Timed> hybridResult = inMethodTransaction(method, hnswIndex, + () -> timed(() -> hybridService.search(vector, question.text(), modelId, permittedIds, topK))); + List runHybridIds = chunkIds(hybridResult.value()); + + vectorMillis.add(vectorResult.millis()); + hybridMillis.add(hybridResult.millis()); + if (run == 0) { + sqlChunkIds = runSqlIds; + vectorFinalIds = runVectorIds; + hybridFinalIds = runHybridIds; + hybridLexicalRows = recorder.lexicalRows(); + } else { + stable &= sqlChunkIds.equals(runSqlIds) && vectorFinalIds.equals(runVectorIds) + && hybridFinalIds.equals(runHybridIds); + } + } + return new QuestionOutcome(sqlChunkIds, vectorFinalIds, hybridFinalIds, hybridLexicalRows, + List.copyOf(vectorMillis), List.copyOf(hybridMillis), stable, plan, rawPlan); + } + + /** + * 서비스 호출과 같은 Transaction에서 방식을 바꾸고 항상 Rollback한다. Exact의 Index 삭제도 Rollback으로 되돌린다. + * + *

Plan Cache 모드는 매번 명시한다. 기본값({@code auto})이면 같은 SQL을 5번 넘게 실행한 Connection에서 Generic Plan으로 + * 바뀔 수 있어 실행 순서에 따라 지연이 달라지기 때문이다. HNSW 강제 방식의 {@code enable_sort=off}는 단어 갈래 조회에 + * 영향을 주지 않도록 Repository Proxy가 findTopK 직전에만 켜고 직후에 되돌린다. + */ + private T inMethodTransaction(Method method, HnswIndex hnswIndex, Supplier work) { + TransactionTemplate template = new TransactionTemplate(transactionManager); + return template.execute(status -> { + status.setRollbackOnly(); + jdbcTemplate.execute("SET LOCAL plan_cache_mode = " + method.planCacheMode().setting()); + if (method == Method.EXACT) { + jdbcTemplate.execute("DROP INDEX " + TEST_SCHEMA + "." + hnswIndex.name()); + } else { + jdbcTemplate.execute("SET LOCAL hnsw.ef_search = " + method.efSearch()); + jdbcTemplate.execute("SET LOCAL hnsw.iterative_scan = " + method.iterativeScan()); + } + currentMethod = method; + try { + return work.get(); + } finally { + currentMethod = null; + } + }); + } + + /** + * HNSW 강제 방식이면 findTopK 실행 동안만 정렬 계획을 밀어낸다. {@code enabled=false}가 강제 시작, {@code true}가 원복이다. + */ + private void setSortForTopK(Method method, boolean enabled) { + if (method != null && method.forceHnsw()) { + jdbcTemplate.execute("SET LOCAL enable_sort = " + (enabled ? "on" : "off")); + } + } + + /** + * JdbcTemplate의 {@code SET LOCAL}이 JPA Native Query와 같은 Connection·Transaction에 적용되는지 먼저 확인하고, + * 측정에 영향을 주는 PostgreSQL·pgvector 설정값을 기록한다. + */ + private Map verifySessionSettings() { + TransactionTemplate template = new TransactionTemplate(transactionManager); + return template.execute(status -> { + status.setRollbackOnly(); + jdbcTemplate.queryForObject("SELECT CAST('[1]' AS vector)::text", String.class); + jdbcTemplate.execute("SET LOCAL hnsw.ef_search = 123"); + Object observed = entityManager.createNativeQuery("SHOW hnsw.ef_search").getSingleResult(); + assertThat(observed).hasToString("123"); + jdbcTemplate.execute("RESET hnsw.ef_search"); + + Map settings = new LinkedHashMap<>(); + for (String name : List.of("hnsw.ef_search", "hnsw.iterative_scan", "hnsw.max_scan_tuples", + "hnsw.scan_mem_multiplier", "shared_buffers", "work_mem", "maintenance_work_mem", + "effective_cache_size", "random_page_cost", "max_parallel_workers_per_gather", + "max_parallel_maintenance_workers", "plan_cache_mode", "jit")) { + settings.put(name, jdbcTemplate.queryForObject("SHOW " + name, String.class)); + } + return settings; + }); + } + + private String jdbcDriverVersion() throws SQLException { + try (Connection connection = dataSource.getConnection()) { + return connection.getMetaData().getDriverVersion(); + } + } + + private HnswIndex currentHnswIndex() { + List indexes = jdbcTemplate.query(""" + SELECT indexname, indexdef FROM pg_indexes + WHERE schemaname = current_schema() AND tablename = 'embeddings' AND indexdef ILIKE '%USING hnsw%' + """, (resultSet, rowNumber) -> new HnswIndex(resultSet.getString(1), resultSet.getString(2))); + if (!indexes.isEmpty()) { + return indexes.get(0); + } + // 이전 실행이 Index를 지운 채 중단됐을 때를 위한 V32 정의 + return new HnswIndex(FALLBACK_HNSW_INDEX, "CREATE INDEX " + FALLBACK_HNSW_INDEX + " ON " + TEST_SCHEMA + + ".embeddings USING hnsw (vector vector_cosine_ops)"); + } + + /** + * 실제 문서와 합성 문서를 COPY로 적재하고, V32와 같은 정의로 HNSW Index를 다시 만들며 생성 시간을 잰다. + */ + private LoadResult loadDataset(PermissionRecallDataset dataset, Long modelId, HnswIndex hnswIndex) + throws Exception { + jdbcTemplate.execute("TRUNCATE TABLE embeddings, document_chunks, document_versions, documents, users " + + "RESTART IDENTITY CASCADE"); + jdbcTemplate.execute("DROP INDEX IF EXISTS " + TEST_SCHEMA + "." + hnswIndex.name()); + Long ownerId = jdbcTemplate.queryForObject(""" + INSERT INTO users (email, password_hash, name, status, created_at, updated_at) + VALUES ('permission-recall-545@example.com', 'password-hash', 'Permission Recall 545', 'ACTIVE', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id + """, Long.class); + + long loadStartedAt = System.nanoTime(); + try (Connection connection = dataSource.getConnection()) { + PGConnection pgConnection = connection.unwrap(PGConnection.class); + + // 1. 문서·버전 id를 같게 두고 현재 버전을 연결한다. 모두 INDEXED·삭제 안 됨 상태다. + copy(pgConnection, "documents (id, owner_user_id, title, document_type, source_type, status, visibility)", + dataset.documents(), spec -> List.of(id(spec), ownerId, title(dataset, spec), + dataset.source(spec).documentType(), "UPLOAD", "INDEXED", "PRIVATE")); + copy(pgConnection, "document_versions (id, document_id, version_no, title_snapshot, content_type, status, " + + "created_by)", + dataset.documents(), spec -> List.of(id(spec), id(spec), 1, title(dataset, spec), "text/plain", + "INDEXED", ownerId)); + try (var statement = connection.createStatement()) { + statement.execute("UPDATE documents SET current_version_id = id"); + } + + // 2. Chunk 본문은 원본 그대로 복제하고, Embedding은 실제 값 또는 합성 Vector를 그 자리에서 계산해 보낸다. + CopyIn chunks = pgConnection.getCopyAPI().copyIn("COPY document_chunks (id, document_version_id, " + + "chunk_index, chunk_text, token_count, char_start, char_end, page_no, section_title, content_hash, " + + "metadata_json) FROM STDIN WITH (FORMAT csv)"); + CopyIn embeddings = null; + long chunkId = 0; + try { + for (DocumentSpec spec : dataset.documents()) { + for (RealChunk chunk : dataset.source(spec).chunks()) { + chunkId++; + writeRow(chunks, List.of(chunkId, id(spec), chunk.chunkIndex(), chunk.chunkText(), + chunk.tokenCount(), chunk.charStart(), chunk.charEnd(), nullable(chunk.pageNo()), + nullable(chunk.sectionTitle()), chunk.contentHash(), nullable(chunk.metadataJson()))); + } + } + chunks.endCopy(); + + embeddings = pgConnection.getCopyAPI().copyIn("COPY embeddings (id, chunk_id, document_id, " + + "document_version_id, embedding_model_id, vector, dimension, vector_hash, status) " + + "FROM STDIN WITH (FORMAT csv)"); + chunkId = 0; + for (DocumentSpec spec : dataset.documents()) { + List sourceChunks = dataset.source(spec).chunks(); + for (int index = 0; index < sourceChunks.size(); index++) { + chunkId++; + float[] vector = dataset.vector(spec, index); + writeRow(embeddings, List.of(chunkId, chunkId, id(spec), id(spec), modelId, + vectorString(vector), VECTOR_DIMENSION, EmbeddingVectorSupport.calculateHash(vector), + "ACTIVE")); + } + } + embeddings.endCopy(); + } finally { + if (chunks.isActive()) { + chunks.cancelCopy(); + } + if (embeddings != null && embeddings.isActive()) { + embeddings.cancelCopy(); + } + } + } + jdbcTemplate.execute("ANALYZE documents"); + jdbcTemplate.execute("ANALYZE document_versions"); + jdbcTemplate.execute("ANALYZE document_chunks"); + double loadSeconds = (System.nanoTime() - loadStartedAt) / 1_000_000_000.0; + + // 3. V32에서 만든 것과 같은 정의(m·ef_construction 기본값)로 Index를 다시 만든다. + long indexStartedAt = System.nanoTime(); + jdbcTemplate.execute(hnswIndex.definition()); + double indexBuildSeconds = (System.nanoTime() - indexStartedAt) / 1_000_000_000.0; + jdbcTemplate.execute("ANALYZE embeddings"); + + return new LoadResult( + loadSeconds, + indexBuildSeconds, + jdbcTemplate.queryForObject("SELECT pg_relation_size('embeddings')", Long.class), + jdbcTemplate.queryForObject("SELECT pg_relation_size('" + hnswIndex.name() + "')", Long.class), + jdbcTemplate.queryForObject("SELECT pg_total_relation_size('embeddings')", Long.class) + ); + } + + private static long id(DocumentSpec spec) { + return spec.index() + 1L; + } + + private static String title(PermissionRecallDataset dataset, DocumentSpec spec) { + String title = dataset.source(spec).title(); + String decorated = spec.synthetic() ? title + " · 합성 " + spec.index() : title; + return decorated.length() > 480 ? decorated.substring(0, 480) : decorated; + } + + private static Object nullable(Object value) { + return value == null ? CsvNull.INSTANCE : value; + } + + private static void copy(PGConnection connection, String target, List items, + Function> columns) throws Exception { + CopyIn copyIn = connection.getCopyAPI().copyIn("COPY " + target + " FROM STDIN WITH (FORMAT csv)"); + try { + for (T item : items) { + writeRow(copyIn, columns.apply(item)); + } + copyIn.endCopy(); + } finally { + if (copyIn.isActive()) { + copyIn.cancelCopy(); + } + } + } + + private static void writeRow(CopyIn copyIn, List values) throws Exception { + StringBuilder line = new StringBuilder(); + for (int index = 0; index < values.size(); index++) { + if (index > 0) { + line.append(','); + } + Object value = values.get(index); + if (value instanceof CsvNull) { + continue; + } + if (value instanceof Number) { + line.append(value); + } else { + line.append('"').append(value.toString().replace("\"", "\"\"")).append('"'); + } + } + byte[] bytes = line.append('\n').toString().getBytes(StandardCharsets.UTF_8); + copyIn.writeToCopy(bytes, 0, bytes.length); + } + + /** CSV에서 따옴표 없는 빈 칸(NULL)으로 쓸 값. */ + private enum CsvNull { INSTANCE } + + private PermissionRecallCorpus loadCorpus(Configuration configuration) throws IOException { + return PermissionRecallCorpus.loadOrBuild( + configuration.repositoryRoot(), + configuration.corpusCachePath(), + new PermissionRecallCorpus.Pipeline( + documentParserRegistry, fixedSizeChunker, adaptiveEmbeddingBatchPlanner, embeddingClient), + objectMapper + ); + } + + private List loadQuestions(Configuration configuration) throws IOException { + JsonNode root = objectMapper.readTree(configuration.questionsPath().toFile()); + List questions = new ArrayList<>(); + for (JsonNode node : root.path("questions")) { + questions.add(new Question(node.path("id").asText(), node.path("source").asText(), + node.path("question").asText())); + } + int limit = configuration.questionLimit() > 0 ? configuration.questionLimit() : questions.size(); + return List.copyOf(questions.subList(0, Math.min(limit, questions.size()))); + } + + /** + * 운영 첫 질문과 같은 경로(대화 문맥 없이 원문 질문, 단건 Embedding API)로 질문 Vector를 만들고 캐시한다. + */ + private Map loadQuestionVectors(Configuration configuration, List questions) + throws IOException { + Map vectors = new LinkedHashMap<>(); + Path cachePath = configuration.questionCachePath(); + if (Files.exists(cachePath)) { + JsonNode cached = objectMapper.readTree(cachePath.toFile()); + cached.path("vectors").fields().forEachRemaining(entry -> + vectors.put(entry.getKey(), objectMapper.convertValue(entry.getValue(), float[].class))); + } + boolean changed = false; + for (Question question : questions) { + if (!vectors.containsKey(question.text())) { + vectors.put(question.text(), embeddingClient.embed(question.text())); + changed = true; + } + } + if (changed) { + writeJson(cachePath, Map.of("vectors", vectors)); + } + return vectors; + } + + private void writeJson(Path path, Object value) throws IOException { + Files.createDirectories(path.toAbsolutePath().getParent()); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), value); + } + + /** + * 운영 Repository Bean에 위임하면서 findTopK·단어 갈래 조회의 반환 행을 기록하는 Proxy를 만든다. + */ + private VectorSearchRepository countingRepository(Recorder recorder) { + return (VectorSearchRepository) Proxy.newProxyInstance( + VectorSearchRepository.class.getClassLoader(), + new Class[] {VectorSearchRepository.class}, + (proxy, method, arguments) -> { + boolean topK = "findTopK".equals(method.getName()); + Object result; + try { + if (topK) { + setSortForTopK(currentMethod, false); + } + result = method.invoke(vectorSearchRepository, arguments); + } catch (InvocationTargetException exception) { + throw exception.getCause(); + } finally { + if (topK) { + setSortForTopK(currentMethod, true); + } + } + switch (method.getName()) { + case "findTopK" -> recorder.recordTopK((List) result); + case "findLexicalCandidates" -> recorder.recordLexical((List) result); + default -> { } + } + return result; + }); + } + + private static String topKSql() { + try { + return VectorSearchRepository.class + .getMethod("findTopK", String.class, Long.class, List.class, int.class) + .getAnnotation(Query.class) + .value(); + } catch (NoSuchMethodException exception) { + throw new IllegalStateException(exception); + } + } + + /** 운영 VectorSearchQueryService와 같은 규칙(StringBuilder의 float 표기)으로 Vector 문자열을 만든다. */ + static String vectorString(float[] vector) { + StringBuilder builder = new StringBuilder(vector.length * 12).append('['); + for (int index = 0; index < vector.length; index++) { + if (index > 0) { + builder.append(','); + } + builder.append(vector[index]); + } + return builder.append(']').toString(); + } + + private static List chunkIds(List candidates) { + return candidates.stream().map(VectorSearchCandidate::chunkId).toList(); + } + + private static Timed timed(Supplier work) { + long startedAt = System.nanoTime(); + T value = work.get(); + return new Timed<>(value, (System.nanoTime() - startedAt) / 1_000_000.0); + } + + /** Plan Cache 모드. */ + enum PlanCacheMode { + CUSTOM("force_custom_plan"), + GENERIC("force_generic_plan"); + + private final String setting; + + PlanCacheMode(String setting) { + this.setting = setting; + } + + String setting() { + return setting; + } + } + + /** + * 측정 방식. + * + *
    + *
  • {@code exact}: HNSW Index를 지운 상태. Recall 정답이다.
    • + *
  • {@code planner_default}·{@code planner_default_generic}: Index가 있고 설정은 기본값. 운영이 실제로 고르는 계획을 + * Custom Plan과 Generic Plan(Prepared Statement 재사용 후)으로 잰다.
    • + *
  • {@code hnsw_*}·{@code iterative_*}: {@code enable_sort=off}로 HNSW를 쓰게 한 뒤 ef_search·iterative_scan을 바꾼다.
    • + *
+ */ + enum Method { + EXACT("exact", PlanCacheMode.CUSTOM, false, 0, "off"), + PLANNER_DEFAULT("planner_default", PlanCacheMode.CUSTOM, false, 40, "off"), + PLANNER_DEFAULT_GENERIC("planner_default_generic", PlanCacheMode.GENERIC, false, 40, "off"), + HNSW_EF40("hnsw_ef40", PlanCacheMode.CUSTOM, true, 40, "off"), + HNSW_EF100("hnsw_ef100", PlanCacheMode.CUSTOM, true, 100, "off"), + HNSW_EF200("hnsw_ef200", PlanCacheMode.CUSTOM, true, 200, "off"), + ITERATIVE_RELAXED("iterative_relaxed", PlanCacheMode.CUSTOM, true, 40, "relaxed_order"), + ITERATIVE_STRICT("iterative_strict", PlanCacheMode.CUSTOM, true, 40, "strict_order"); + + private final String label; + private final PlanCacheMode planCacheMode; + private final boolean forceHnsw; + private final int efSearch; + private final String iterativeScan; + + Method(String label, PlanCacheMode planCacheMode, boolean forceHnsw, int efSearch, String iterativeScan) { + this.label = label; + this.planCacheMode = planCacheMode; + this.forceHnsw = forceHnsw; + this.efSearch = efSearch; + this.iterativeScan = iterativeScan; + } + + String label() { + return label; + } + + PlanCacheMode planCacheMode() { + return planCacheMode; + } + + boolean forceHnsw() { + return forceHnsw; + } + + int efSearch() { + return efSearch; + } + + String iterativeScan() { + return iterativeScan; + } + + static Method fromLabel(String label) { + return Arrays.stream(values()).filter(method -> method.label.equals(label)).findFirst() + .orElseThrow(() -> new IllegalArgumentException("알 수 없는 방식: " + label)); + } + } + + /** 서비스 호출 한 번이 Repository에서 받은 행. */ + static final class Recorder { + private List topKChunkIds = List.of(); + private int lexicalRows; + + void reset() { + topKChunkIds = List.of(); + lexicalRows = 0; + } + + void recordTopK(List rows) { + topKChunkIds = rows.stream().map(row -> ((VectorSearchRow) row).getChunkId()).toList(); + } + + void recordLexical(List rows) { + lexicalRows = rows.size(); + } + + List topKChunkIds() { + return topKChunkIds; + } + + int lexicalRows() { + return lexicalRows; + } + } + + /** 시스템 속성({@code vector.search.performance.permission.*})으로 정하는 실행 설정. 기본값은 이 시험(#545) 측정 조건이다. */ + record Configuration( + Path repositoryRoot, + Path corpusCachePath, + Path questionsPath, + Path questionCachePath, + Path outputPath, + Path planOutputPath, + List scales, + List ratios, + List topKs, + List methods, + int warmUpRuns, + int measuredRuns, + int genericWarmUpRuns, + int genericMeasuredRuns, + int questionLimit, + long seed + ) { + + static Configuration fromSystemProperties() { + return new Configuration( + Path.of(property("repository-root", "..")), + Path.of(property("corpus-cache", "build/reports/permission-recall/real-corpus.json")), + Path.of(property("questions", "../docs/test-results/evidence/issue-545/questions.json")), + Path.of(property("question-cache", "build/reports/permission-recall/question-vectors.json")), + Path.of(property("output", "build/reports/permission-recall/results.json")), + Path.of(property("plan-output", "build/reports/permission-recall/plans.json")), + split(property("scales", "real,50000")), + split(property("ratios", "1.0,0.1,0.01,0.001")).stream().map(Double::parseDouble).toList(), + split(property("top-ks", "5,10,20")).stream().map(Integer::parseInt).toList(), + split(property("methods", "exact,planner_default,planner_default_generic," + + "hnsw_ef40,hnsw_ef100,hnsw_ef200,iterative_relaxed,iterative_strict")) + .stream().map(Method::fromLabel).toList(), + Integer.parseInt(property("warm-up", "1")), + Integer.parseInt(property("measured-runs", "3")), + Integer.parseInt(property("generic-warm-up", "0")), + Integer.parseInt(property("generic-measured-runs", "1")), + Integer.parseInt(property("question-limit", "0")), + Long.parseLong(property("seed", "545")) + ); + } + + private static String property(String name, String defaultValue) { + return System.getProperty(PROPERTY_PREFIX + name, defaultValue); + } + + private static List split(String value) { + return Arrays.stream(value.split(",")).map(String::strip).filter(item -> !item.isEmpty()).toList(); + } + } + + /** 고정 질문 한 건과 그 질문을 만든 원본 문서 경로. */ + record Question(String id, String source, String text) { + } + + /** HNSW Index 이름과 다시 만들 때 쓰는 정의(V32와 같음). */ + record HnswIndex(String name, String definition) { + } + + /** 서비스 호출 결과와 걸린 시간(ms). */ + record Timed(T value, double millis) { + } + + /** 데이터 적재·Index 생성 시간과 크기. */ + record LoadResult(double loadSeconds, double indexBuildSeconds, long tableBytes, long indexBytes, + long totalBytes) { + } + + /** 질문 한 건 × 방식 한 개의 SQL 후보·최종 결과·지연·실행 계획. ID 목록은 첫 측정 결과다. */ + record QuestionOutcome( + List sqlChunkIds, + List vectorFinalIds, + List hybridFinalIds, + int hybridLexicalRows, + List vectorMillis, + List hybridMillis, + boolean stable, + PlanSummary plan, + String rawPlan + ) { + } + + /** 조합마다 첫 질문의 EXPLAIN 원문(별도 파일로 저장). */ + record PlanSample(String scale, double ratio, int topK, String method, String questionId, String plan) { + } + + /** 허용 비율별 실제 허용 문서·청크 수. */ + record RatioInfo(double ratio, int permittedDocuments, int permittedChunks) { + } + + /** 규모 한 개의 데이터 구성·합성 비율·Index 비용. */ + record ScaleResult( + String scale, + int documents, + int realDocuments, + int syntheticDocuments, + int chunks, + int realChunks, + double syntheticChunkRatio, + double syntheticSimilarityLow, + double syntheticSimilarityHigh, + double loadSeconds, + double indexBuildSeconds, + long tableBytes, + long indexBytes, + long totalBytes, + List ratios + ) { + } + + /** 측정 당시 운영 검색 설정값(application.yml 기본값). */ + record SearchSettings( + double minSimilarity, + int maxChunksPerDocument, + int candidatePoolMultiplier, + boolean hybridEnabled, + double hybridVectorMinSimilarity, + double hybridMinCoverage, + double hybridCoverageWeight, + int hybridLexicalCandidatePoolMultiplier + ) { + } + + /** 결과 파일(results.json) 전체 구조. */ + record Report( + String measuredAt, + String postgresVersion, + String pgvectorVersion, + Map postgresSettings, + String osArch, + String javaVersion, + String jdbcDriverVersion, + String embeddingModel, + int corpusDocuments, + int corpusChunks, + List skippedSources, + double corpusEmbeddingSeconds, + Distribution crossDocumentNearestSimilarity, + SearchSettings searchSettings, + String hnswIndexDefinition, + Configuration configuration, + List scales, + List summaries, + List rows + ) { + } + + /** 실행 계획에서 HNSW 사용 여부와 필터로 버려진 행 등 판정에 필요한 값만 추린 것. */ + record PlanSummary( + boolean usesHnsw, + List nodeTypes, + List indexNames, + Long hnswActualRows, + Long hnswRowsRemovedByFilter, + long resultRows, + double planningMillis, + double executionMillis, + long sharedHitBlocks, + long sharedReadBlocks + ) { + + static PlanSummary parse(JsonNode explain, String hnswIndexName) { + JsonNode root = explain.get(0); + JsonNode plan = root.path("Plan"); + List nodeTypes = new ArrayList<>(); + List indexNames = new ArrayList<>(); + List hnswNodes = new ArrayList<>(); + collect(plan, hnswIndexName, nodeTypes, indexNames, hnswNodes); + JsonNode hnswNode = hnswNodes.isEmpty() ? null : hnswNodes.get(0); + return new PlanSummary( + hnswNode != null, + List.copyOf(nodeTypes), + List.copyOf(indexNames), + hnswNode == null ? null : hnswNode.path("Actual Rows").asLong(), + hnswNode == null ? null : hnswNode.path("Rows Removed by Filter").asLong(0), + plan.path("Actual Rows").asLong(), + root.path("Planning Time").asDouble(), + root.path("Execution Time").asDouble(), + plan.path("Shared Hit Blocks").asLong(), + plan.path("Shared Read Blocks").asLong() + ); + } + + private static void collect(JsonNode node, String hnswIndexName, List nodeTypes, + List indexNames, List hnswNodes) { + nodeTypes.add(node.path("Node Type").asText()); + if (node.has("Index Name")) { + String indexName = node.path("Index Name").asText(); + indexNames.add(indexName); + if (indexName.equals(hnswIndexName)) { + hnswNodes.add(node); + } + } + for (JsonNode child : node.path("Plans")) { + collect(child, hnswIndexName, nodeTypes, indexNames, hnswNodes); + } + } + } + + /** 질문 한 건 × 조합 한 개의 측정 행. Recall은 같은 조건의 Exact 결과를 정답으로 계산한다. */ + record MeasurementRow( + String scale, + double ratio, + int permittedDocuments, + int permittedChunks, + int topK, + int candidateLimit, + String method, + String questionId, + String questionSource, + int sqlRows, + int vectorFinal, + int hybridFinal, + int hybridLexicalRows, + int exactSqlRows, + int exactVectorFinal, + int exactHybridFinal, + Double sqlRecall, + Double vectorFinalRecall, + Double hybridFinalRecall, + List vectorMillis, + List hybridMillis, + boolean stable, + PlanSummary plan + ) { + + static MeasurementRow of(String scale, double ratio, int permittedDocuments, int permittedChunks, int topK, + int candidateLimit, Method method, Question question, QuestionOutcome outcome, QuestionOutcome exact) { + return new MeasurementRow( + scale, ratio, permittedDocuments, permittedChunks, topK, candidateLimit, method.label(), + question.id(), question.source(), + outcome.sqlChunkIds().size(), outcome.vectorFinalIds().size(), outcome.hybridFinalIds().size(), + outcome.hybridLexicalRows(), + exact.sqlChunkIds().size(), exact.vectorFinalIds().size(), exact.hybridFinalIds().size(), + PermissionRecallMetrics.recall(outcome.sqlChunkIds(), exact.sqlChunkIds()), + PermissionRecallMetrics.recall(outcome.vectorFinalIds(), exact.vectorFinalIds()), + PermissionRecallMetrics.recall(outcome.hybridFinalIds(), exact.hybridFinalIds()), + outcome.vectorMillis(), outcome.hybridMillis(), outcome.stable(), outcome.plan() + ); + } + } + + /** 분포 요약(오름차순 정렬 후 최근접 순위 백분위). */ + record Distribution(int count, double min, double p10, double p50, double p90, double p95, double max, + double mean) { + + static Distribution of(double[] values) { + double[] sorted = values.clone(); + Arrays.sort(sorted); + return new Distribution(sorted.length, sorted[0], + PermissionRecallMetrics.percentile(sorted, 0.10), PermissionRecallMetrics.percentile(sorted, 0.50), + PermissionRecallMetrics.percentile(sorted, 0.90), PermissionRecallMetrics.percentile(sorted, 0.95), + sorted[sorted.length - 1], Arrays.stream(sorted).average().orElse(Double.NaN)); + } + } + + /** 규모 × 비율 × topK × 방식 한 조합의 질문 전체 요약. */ + record CombinationSummary( + String scale, + double ratio, + int permittedDocuments, + int permittedChunks, + int topK, + int candidateLimit, + String method, + int questions, + double sqlRowsMean, + int sqlRowsMin, + int sqlRowsMax, + double vectorFinalMean, + int vectorShortQuestions, + double hybridFinalMean, + int hybridShortQuestions, + double exactVectorFinalMean, + double exactHybridFinalMean, + Double sqlRecallMean, + Double sqlRecallMin, + Double vectorFinalRecallMean, + Double vectorFinalRecallMin, + Double hybridFinalRecallMean, + Double hybridFinalRecallMin, + double vectorP50Millis, + double vectorP95Millis, + double hybridP50Millis, + double hybridP95Millis, + int hnswPlanQuestions, + int unstableQuestions + ) { + + static List summarize(List rows) { + Map> groups = new LinkedHashMap<>(); + for (MeasurementRow row : rows) { + groups.computeIfAbsent(row.scale() + "|" + row.ratio() + "|" + row.topK() + "|" + row.method(), + key -> new ArrayList<>()).add(row); + } + List summaries = new ArrayList<>(); + for (List group : groups.values()) { + MeasurementRow first = group.get(0); + summaries.add(new CombinationSummary( + first.scale(), first.ratio(), first.permittedDocuments(), first.permittedChunks(), first.topK(), + first.candidateLimit(), first.method(), group.size(), + group.stream().mapToInt(MeasurementRow::sqlRows).average().orElse(0), + group.stream().mapToInt(MeasurementRow::sqlRows).min().orElse(0), + group.stream().mapToInt(MeasurementRow::sqlRows).max().orElse(0), + group.stream().mapToInt(MeasurementRow::vectorFinal).average().orElse(0), + (int) group.stream().filter(row -> row.vectorFinal() < row.topK()).count(), + group.stream().mapToInt(MeasurementRow::hybridFinal).average().orElse(0), + (int) group.stream().filter(row -> row.hybridFinal() < row.topK()).count(), + group.stream().mapToInt(MeasurementRow::exactVectorFinal).average().orElse(0), + group.stream().mapToInt(MeasurementRow::exactHybridFinal).average().orElse(0), + PermissionRecallMetrics.mean(group.stream().map(MeasurementRow::sqlRecall).toList()), + PermissionRecallMetrics.min(group.stream().map(MeasurementRow::sqlRecall).toList()), + PermissionRecallMetrics.mean(group.stream().map(MeasurementRow::vectorFinalRecall).toList()), + PermissionRecallMetrics.min(group.stream().map(MeasurementRow::vectorFinalRecall).toList()), + PermissionRecallMetrics.mean(group.stream().map(MeasurementRow::hybridFinalRecall).toList()), + PermissionRecallMetrics.min(group.stream().map(MeasurementRow::hybridFinalRecall).toList()), + PermissionRecallMetrics.percentile(latencies(group, true), 0.50), + PermissionRecallMetrics.percentile(latencies(group, true), 0.95), + PermissionRecallMetrics.percentile(latencies(group, false), 0.50), + PermissionRecallMetrics.percentile(latencies(group, false), 0.95), + (int) group.stream().filter(row -> row.plan().usesHnsw()).count(), + (int) group.stream().filter(row -> !row.stable()).count() + )); + } + return List.copyOf(summaries); + } + + private static double[] latencies(List group, boolean vector) { + return group.stream() + .flatMap(row -> (vector ? row.vectorMillis() : row.hybridMillis()).stream()) + .mapToDouble(Double::doubleValue) + .sorted() + .toArray(); + } + } +} diff --git a/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallCorpus.java b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallCorpus.java new file mode 100644 index 00000000..4e6495bd --- /dev/null +++ b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallCorpus.java @@ -0,0 +1,269 @@ +package com.opensource.docgrid.domain.search.benchmark; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Stream; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opensource.docgrid.domain.document.enums.DocumentType; +import com.opensource.docgrid.domain.document.service.DocumentChunkDraft; +import com.opensource.docgrid.domain.document.service.DocumentParserRegistry; +import com.opensource.docgrid.domain.document.service.FixedSizeChunker; +import com.opensource.docgrid.domain.document.service.ParsedDocument; +import com.opensource.docgrid.domain.document.service.SectionPath; +import com.opensource.docgrid.domain.embedding.client.EmbeddingClient; +import com.opensource.docgrid.domain.embedding.dto.response.EmbedBatchServerResponse; +import com.opensource.docgrid.domain.embedding.service.AdaptiveEmbeddingBatchPlanner; +import com.opensource.docgrid.domain.embedding.service.command.DocumentEmbeddingTransactionService.ChunkSnapshot; + +/** + * 권한 필터 검색 누락 시험(#545)에 쓰는 실제 문서 Chunk와 BGE-M3 Embedding 묶음. + * + *

저장소 안의 팀 작성 문서(예제 문서 15건, {@code docs/design}·{@code docs/test-results} 최상위 Markdown)를 + * 운영과 같은 Parser·Chunker·Embedding 입력 규칙(Section 경로 포함)·Batch 계획으로 처리해 만든다. + * Embedding 서버 호출은 오래 걸리므로 결과를 JSON 파일로 캐시하고, 원본 파일 SHA-256이 모두 같으면 다시 쓴다. + * DB 적재와 합성 확장은 하지 않는다. 그 역할은 {@link PermissionRecallDataset}과 {@link PermissionRecallBenchmark}가 맡는다. + * + * @param embeddingModel Embedding 서버가 응답한 모델 이름 + * @param createdAt 캐시 생성 시각(ISO-8601) + * @param embeddingSeconds 전체 Chunk Embedding에 걸린 시간 + * @param skippedSources 파싱·Chunking에 실패해 제외한 원본 경로와 사유 + * @param documents 원본 경로 순으로 정렬한 실제 문서 + */ +record PermissionRecallCorpus( + String embeddingModel, + String createdAt, + double embeddingSeconds, + List skippedSources, + List documents +) { + + private static final String DEMO_CORPUS_DIRECTORY = "examples/demo-corpus"; + private static final List MARKDOWN_DIRECTORIES = List.of("docs/design", "docs/test-results"); + private static final Map DEMO_TYPES = Map.of( + "pdf", DocumentType.PDF, + "docx", DocumentType.DOCX, + "hwpx", DocumentType.HWPX + ); + + /** + * 실제 문서 한 건과 그 Chunk. + * + * @param sourcePath 저장소 루트 기준 원본 경로 + * @param sourceSha256 원본 파일 SHA-256 + * @param title 예제 문서는 manifest 제목, Markdown은 첫 번째 {@code # } 제목(없으면 파일 이름) + * @param documentType 운영 Parser 선택에 쓴 문서 형식 + * @param chunks Chunk Index 순 Chunk + */ + record RealDocument( + String sourcePath, + String sourceSha256, + String title, + String documentType, + List chunks + ) { + } + + /** + * 운영 Chunker가 만든 Chunk와 운영 Embedding 입력으로 만든 Vector. + * + * @param vector Embedding 서버가 돌려준 값 그대로(정규화하지 않음) + */ + record RealChunk( + int chunkIndex, + String chunkText, + int tokenCount, + int charStart, + int charEnd, + Integer pageNo, + String sectionTitle, + String contentHash, + String metadataJson, + float[] vector + ) { + } + + /** 원본 파일 한 건의 경로·형식·제목. */ + record SourceFile(Path path, String relativePath, DocumentType documentType, String title) { + } + + int chunkCount() { + return documents.stream().mapToInt(document -> document.chunks().size()).sum(); + } + + /** + * 캐시가 있고 원본 SHA-256이 모두 같으면 캐시를 읽고, 아니면 운영 처리 경로로 다시 만들어 저장한다. + */ + static PermissionRecallCorpus loadOrBuild( + Path repositoryRoot, + Path cachePath, + Pipeline pipeline, + ObjectMapper objectMapper + ) throws IOException { + List sources = listSources(repositoryRoot, objectMapper); + if (Files.exists(cachePath)) { + PermissionRecallCorpus cached = objectMapper.readValue(cachePath.toFile(), PermissionRecallCorpus.class); + if (cached.matches(sources)) { + return cached; + } + } + PermissionRecallCorpus built = build(sources, pipeline); + Files.createDirectories(cachePath.toAbsolutePath().getParent()); + objectMapper.writeValue(cachePath.toFile(), built); + return built; + } + + /** + * 시험 대상 원본 목록을 정렬된 순서로 만든다. 예제 문서는 manifest 제목을 쓴다. + */ + static List listSources(Path repositoryRoot, ObjectMapper objectMapper) throws IOException { + List sources = new ArrayList<>(); + + // 1. 예제 문서: 가상 회사 규정 PDF·DOCX·HWPX 15건 (팀 작성, 저작권 문제 없음) + Path demoDirectory = repositoryRoot.resolve(DEMO_CORPUS_DIRECTORY); + JsonNode manifest = objectMapper.readTree(demoDirectory.resolve("manifest.json").toFile()); + for (JsonNode entry : manifest.path("documents")) { + String file = entry.path("file").asText(); + String extension = file.substring(file.lastIndexOf('.') + 1).toLowerCase(Locale.ROOT); + sources.add(new SourceFile( + demoDirectory.resolve(file), + DEMO_CORPUS_DIRECTORY + "/" + file, + DEMO_TYPES.get(extension), + entry.path("title").asText() + )); + } + + // 2. 팀 작성 설계·시험 결과 Markdown (하위 evidence 폴더의 원시 로그는 제외) + for (String directory : MARKDOWN_DIRECTORIES) { + try (Stream files = Files.list(repositoryRoot.resolve(directory))) { + for (Path file : files.filter(path -> path.toString().endsWith(".md")).sorted().toList()) { + sources.add(new SourceFile( + file, + directory + "/" + file.getFileName(), + DocumentType.MD, + markdownTitle(file) + )); + } + } + } + return List.copyOf(sources); + } + + private boolean matches(List sources) throws IOException { + int matched = documents.size() + skippedSources.size(); + if (matched != sources.size()) { + return false; + } + Map cachedHashes = new HashMap<>(); + documents.forEach(document -> cachedHashes.put(document.sourcePath(), document.sourceSha256())); + for (SourceFile source : sources) { + String cachedHash = cachedHashes.get(source.relativePath()); + boolean skipped = skippedSources.stream().anyMatch(entry -> entry.startsWith(source.relativePath() + " ")); + if (!skipped && !sha256(Files.readAllBytes(source.path())).equals(cachedHash)) { + return false; + } + } + return true; + } + + private static PermissionRecallCorpus build(List sources, Pipeline pipeline) throws IOException { + List documents = new ArrayList<>(); + List skipped = new ArrayList<>(); + String model = null; + long startedAt = System.nanoTime(); + + for (SourceFile source : sources) { + byte[] content = Files.readAllBytes(source.path()); + List drafts; + try { + // 1. 운영 DocumentParsingService와 같은 순서로 파싱한 뒤 Segment 경계를 지키며 Chunking한다. + ParsedDocument parsed = pipeline.parserRegistry().parse(source.documentType(), content); + drafts = pipeline.chunker().chunk(parsed); + } catch (RuntimeException exception) { + skipped.add(source.relativePath() + " " + exception.getClass().getSimpleName()); + continue; + } + if (drafts.isEmpty()) { + skipped.add(source.relativePath() + " EMPTY"); + continue; + } + + // 2. 운영 Embedding 준비 단계처럼 Section 경로를 붙인 입력을 만들고 같은 Batch 계획으로 나눈다. + List snapshots = new ArrayList<>(drafts.size()); + for (DocumentChunkDraft draft : drafts) { + snapshots.add(new ChunkSnapshot( + (long) draft.chunkIndex(), + draft.chunkIndex(), + draft.chunkText(), + SectionPath.embeddingInput(draft.metadataJson(), draft.chunkText()), + draft.tokenCount(), + draft.contentHash() + )); + } + float[][] vectors = new float[drafts.size()][]; + for (List batch : pipeline.batchPlanner().plan(snapshots)) { + EmbedBatchServerResponse response = pipeline.embeddingClient().embedBatch( + batch.stream().map(ChunkSnapshot::embeddingText).toList(), + batch.size() + ); + model = response.model(); + for (int index = 0; index < batch.size(); index++) { + vectors[batch.get(index).chunkIndex()] = response.embeddings().get(index).vector(); + } + } + + List chunks = new ArrayList<>(drafts.size()); + for (DocumentChunkDraft draft : drafts) { + chunks.add(new RealChunk( + draft.chunkIndex(), draft.chunkText(), draft.tokenCount(), draft.charStart(), draft.charEnd(), + draft.pageNo(), draft.sectionTitle(), draft.contentHash(), draft.metadataJson(), + vectors[draft.chunkIndex()] + )); + } + documents.add(new RealDocument( + source.relativePath(), sha256(content), source.title(), source.documentType().name(), List.copyOf(chunks) + )); + } + + double seconds = (System.nanoTime() - startedAt) / 1_000_000_000.0; + return new PermissionRecallCorpus(model, Instant.now().toString(), seconds, List.copyOf(skipped), + List.copyOf(documents)); + } + + private static String markdownTitle(Path file) throws IOException { + try (Stream lines = Files.lines(file)) { + return lines.filter(line -> line.startsWith("# ")) + .findFirst() + .map(line -> line.substring(2).strip()) + .orElse(file.getFileName().toString()); + } + } + + static String sha256(byte[] content) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException(exception); + } + } + + /** 운영 문서 처리에 쓰는 Spring Bean 묶음. */ + record Pipeline( + DocumentParserRegistry parserRegistry, + FixedSizeChunker chunker, + AdaptiveEmbeddingBatchPlanner batchPlanner, + EmbeddingClient embeddingClient + ) { + } +} diff --git a/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDataset.java b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDataset.java new file mode 100644 index 00000000..52eb3ea0 --- /dev/null +++ b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDataset.java @@ -0,0 +1,217 @@ +package com.opensource.docgrid.domain.search.benchmark; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +import com.opensource.docgrid.domain.search.benchmark.PermissionRecallCorpus.RealChunk; +import com.opensource.docgrid.domain.search.benchmark.PermissionRecallCorpus.RealDocument; + +/** + * 실제 Chunk 묶음을 시험 규모까지 늘리는 결정적 합성 계획과 권한 허용 문서 선택. + * + *

합성 문서 하나는 실제 문서 하나(원본)의 Chunk 본문·Section 정보를 그대로 복제한다. 각 Chunk Vector는 원본 Vector와의 + * 코사인 유사도가 정확히 {@code c}가 되도록, 다른 실제 문서 Chunk Vector에서 원본에 직교하는 성분만 섞어 만든다. + * 섞는 방향이 실제 Embedding에서 오므로 무작위 Gaussian 잡음보다 실제 분포에 가깝게 남는다. {@code c}는 문서마다 + * {@code [similarityLow, similarityHigh]}에서 고르며, 호출 측이 실제 Chunk의 "다른 문서 최근접 유사도" 분포로 정한다. + * + *

Vector는 계획만 들고 있다가 {@link #vector(DocumentSpec, int)} 호출 시 Seed로 다시 계산하므로, 수만 Chunk를 + * 메모리에 올리지 않는다. DB 적재·측정은 하지 않는다. + */ +final class PermissionRecallDataset { + + private final List realDocuments; + private final List unitVectors; + private final List documents; + private final int chunkCount; + + /** + * 시험 데이터의 문서 한 건. + * + * @param sourceDocument 본문을 복제한 실제 문서 Index + * @param mixDocument Vector에 방향을 섞은 다른 실제 문서 Index. 실제 문서면 -1 + * @param similarity 합성 Chunk와 원본 Chunk의 코사인 유사도. 실제 문서면 1 + * @param seed Chunk별 섞을 Chunk를 고르는 Seed + */ + record DocumentSpec(int index, int sourceDocument, int mixDocument, double similarity, long seed) { + + boolean synthetic() { + return mixDocument >= 0; + } + } + + private PermissionRecallDataset(List realDocuments, List documents) { + this.realDocuments = realDocuments; + this.unitVectors = realDocuments.stream() + .map(document -> document.chunks().stream().map(chunk -> normalize(chunk.vector())).toArray(float[][]::new)) + .toList(); + this.documents = List.copyOf(documents); + this.chunkCount = documents.stream() + .mapToInt(spec -> realDocuments.get(spec.sourceDocument()).chunks().size()) + .sum(); + } + + /** + * 실제 문서를 모두 넣고, 전체 Chunk 수가 {@code targetChunks} 이상이 될 때까지 합성 문서를 더한다. + * {@code targetChunks}가 실제 Chunk 수 이하면 실제 문서만 쓴다. + */ + static PermissionRecallDataset expand( + List realDocuments, + int targetChunks, + double similarityLow, + double similarityHigh, + long seed + ) { + if (realDocuments.size() < 2) { + throw new IllegalArgumentException("합성에는 실제 문서가 2건 이상 필요합니다."); + } + List specs = new ArrayList<>(); + int total = 0; + for (int index = 0; index < realDocuments.size(); index++) { + specs.add(new DocumentSpec(index, index, -1, 1.0, 0L)); + total += realDocuments.get(index).chunks().size(); + } + + Random random = new Random(seed); + while (total < targetChunks) { + int source = random.nextInt(realDocuments.size()); + int mix = random.nextInt(realDocuments.size() - 1); + if (mix >= source) { + mix++; + } + double similarity = similarityLow + random.nextDouble() * (similarityHigh - similarityLow); + specs.add(new DocumentSpec(specs.size(), source, mix, similarity, random.nextLong())); + total += realDocuments.get(source).chunks().size(); + } + return new PermissionRecallDataset(realDocuments, specs); + } + + List documents() { + return documents; + } + + int chunkCount() { + return chunkCount; + } + + int realChunkCount() { + return realDocuments.stream().mapToInt(document -> document.chunks().size()).sum(); + } + + RealDocument source(DocumentSpec spec) { + return realDocuments.get(spec.sourceDocument()); + } + + /** + * 문서의 {@code chunkIndex}번째 Chunk Vector. 실제 문서는 서버 응답 값 그대로, 합성 문서는 매번 같은 값으로 다시 계산한다. + */ + float[] vector(DocumentSpec spec, int chunkIndex) { + RealChunk sourceChunk = source(spec).chunks().get(chunkIndex); + if (!spec.synthetic()) { + return sourceChunk.vector(); + } + float[][] mixChunks = unitVectors.get(spec.mixDocument()); + Random random = new Random(spec.seed() * 31 + chunkIndex); + float[] other = mixChunks[random.nextInt(mixChunks.length)]; + return mix(unitVectors.get(spec.sourceDocument())[chunkIndex], other, spec.similarity(), random); + } + + /** + * 단위 Vector {@code source}와의 코사인 유사도가 정확히 {@code similarity}인 단위 Vector를 만든다. + * + *

{@code other}에서 {@code source} 방향을 뺀 직교 성분을 섞는다. 두 Vector가 거의 같은 방향이라 직교 성분이 없으면 + * 무작위 방향으로 대신한다. + */ + static float[] mix(float[] source, float[] other, double similarity, Random fallback) { + double projection = dot(source, other); + double[] orthogonal = new double[source.length]; + double squaredNorm = 0.0; + for (int index = 0; index < source.length; index++) { + orthogonal[index] = other[index] - projection * source[index]; + squaredNorm += orthogonal[index] * orthogonal[index]; + } + if (squaredNorm < 1e-12) { + squaredNorm = 0.0; + for (int index = 0; index < source.length; index++) { + orthogonal[index] = fallback.nextGaussian(); + } + double fallbackProjection = 0.0; + for (int index = 0; index < source.length; index++) { + fallbackProjection += orthogonal[index] * source[index]; + } + for (int index = 0; index < source.length; index++) { + orthogonal[index] -= fallbackProjection * source[index]; + squaredNorm += orthogonal[index] * orthogonal[index]; + } + } + + double orthogonalNorm = Math.sqrt(squaredNorm); + double orthogonalWeight = Math.sqrt(1.0 - similarity * similarity); + float[] mixed = new float[source.length]; + for (int index = 0; index < source.length; index++) { + mixed[index] = (float) (similarity * source[index] + orthogonalWeight * orthogonal[index] / orthogonalNorm); + } + return mixed; + } + + /** + * 실제 Chunk마다 "다른 문서 Chunk 중 가장 높은 코사인 유사도"를 구해 오름차순으로 돌려준다. 합성 유사도 범위를 정하는 근거다. + */ + static double[] crossDocumentNearestSimilarities(List realDocuments) { + List vectors = new ArrayList<>(); + List owners = new ArrayList<>(); + for (int documentIndex = 0; documentIndex < realDocuments.size(); documentIndex++) { + for (RealChunk chunk : realDocuments.get(documentIndex).chunks()) { + vectors.add(normalize(chunk.vector())); + owners.add(documentIndex); + } + } + double[] nearest = new double[vectors.size()]; + Arrays.fill(nearest, -1.0); + for (int left = 0; left < vectors.size(); left++) { + for (int right = left + 1; right < vectors.size(); right++) { + if (owners.get(left).equals(owners.get(right))) { + continue; + } + double similarity = dot(vectors.get(left), vectors.get(right)); + nearest[left] = Math.max(nearest[left], similarity); + nearest[right] = Math.max(nearest[right], similarity); + } + } + Arrays.sort(nearest); + return nearest; + } + + /** + * 문서 Index를 Seed로 섞은 뒤 앞에서부터 {@code ratio} 비율(최소 1건)을 허용 문서로 고른다. + * 같은 Seed면 낮은 비율의 허용 문서가 높은 비율의 허용 문서에 항상 포함된다. + */ + static List permittedDocumentIndexes(int documentCount, double ratio, long seed) { + List shuffled = new ArrayList<>(documentCount); + for (int index = 0; index < documentCount; index++) { + shuffled.add(index); + } + Collections.shuffle(shuffled, new Random(seed)); + int count = Math.max(1, (int) Math.round(documentCount * ratio)); + return List.copyOf(shuffled.subList(0, Math.min(count, documentCount))); + } + + static float[] normalize(float[] vector) { + double norm = Math.sqrt(dot(vector, vector)); + float[] normalized = new float[vector.length]; + for (int index = 0; index < vector.length; index++) { + normalized[index] = (float) (vector[index] / norm); + } + return normalized; + } + + static double dot(float[] left, float[] right) { + double sum = 0.0; + for (int index = 0; index < left.length; index++) { + sum += (double) left[index] * right[index]; + } + return sum; + } +} diff --git a/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallMetrics.java b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallMetrics.java new file mode 100644 index 00000000..08af9418 --- /dev/null +++ b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallMetrics.java @@ -0,0 +1,45 @@ +package com.opensource.docgrid.domain.search.benchmark; + +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * 권한 필터 검색 누락 시험(#545)의 Recall·백분위·평균 계산. + * + *

Recall은 Chunk ID 집합 기준으로 {@code |측정 ∩ 정답| / |정답|}이며, 정답이 비어 있으면 정의하지 않고 {@code null}을 + * 돌려준다. 백분위는 오름차순 정렬된 값에서 최근접 순위(nearest-rank)로 고른다. DB나 Spring에 의존하지 않는다. + */ +final class PermissionRecallMetrics { + + private PermissionRecallMetrics() { + } + + static Double recall(List actual, List expected) { + if (expected.isEmpty()) { + return null; + } + Set expectedIds = new HashSet<>(expected); + long hits = actual.stream().filter(expectedIds::contains).distinct().count(); + return (double) hits / expectedIds.size(); + } + + static double percentile(double[] sortedValues, double quantile) { + if (sortedValues.length == 0) { + return Double.NaN; + } + int rank = (int) Math.ceil(quantile * sortedValues.length); + return sortedValues[Math.max(0, Math.min(sortedValues.length - 1, rank - 1))]; + } + + static Double mean(List values) { + return values.stream().filter(Objects::nonNull).mapToDouble(Double::doubleValue).average() + .stream().boxed().findFirst().orElse(null); + } + + static Double min(List values) { + return values.stream().filter(Objects::nonNull).mapToDouble(Double::doubleValue).min() + .stream().boxed().findFirst().orElse(null); + } +} From dc5d42fc192f10cb505b07bdd316f03fcb6c6ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sat, 17 Oct 2026 18:57:43 +0900 Subject: [PATCH 2/4] =?UTF-8?q?test:=20=EC=8B=9C=ED=97=98=20=ED=95=98?= =?UTF-8?q?=EB=84=A4=EC=8A=A4=20=EA=B3=84=EC=82=B0=20=EB=8B=A8=EC=9C=84=20?= =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80=20(#545)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 합성 Vector의 단위 길이·지정 유사도, 결정적 확장, 중첩되는 허용 문서 선택, Recall·백분위 계산을 DB 없이 검증한다. --- .../PermissionRecallDatasetTest.java | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDatasetTest.java diff --git a/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDatasetTest.java b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDatasetTest.java new file mode 100644 index 00000000..5509fbff --- /dev/null +++ b/backend/src/test/java/com/opensource/docgrid/domain/search/benchmark/PermissionRecallDatasetTest.java @@ -0,0 +1,145 @@ +package com.opensource.docgrid.domain.search.benchmark; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import com.opensource.docgrid.domain.search.benchmark.PermissionRecallCorpus.RealChunk; +import com.opensource.docgrid.domain.search.benchmark.PermissionRecallCorpus.RealDocument; +import com.opensource.docgrid.domain.search.benchmark.PermissionRecallDataset.DocumentSpec; + +/** + * 권한 필터 검색 누락 시험(#545) 하네스의 DB 없는 계산 단위 테스트. + * + *

합성 Vector 수식, 결정적 확장, 허용 문서 선택, Recall·백분위만 검증한다. 측정 자체는 {@link PermissionRecallBenchmark}가 한다. + */ +@DisplayName("권한 필터 검색 누락 시험 데이터·지표 계산 단위 테스트") +class PermissionRecallDatasetTest { + + private static final int DIMENSION = 64; + + @Test + @DisplayName("합성 Vector는 단위 길이이고 원본과의 코사인 유사도가 지정한 값과 같다") + void mix_keepsUnitLengthAndExactSimilarity() { + Random random = new Random(1L); + float[] source = PermissionRecallDataset.normalize(randomVector(random)); + float[] other = PermissionRecallDataset.normalize(randomVector(random)); + + float[] mixed = PermissionRecallDataset.mix(source, other, 0.83, new Random(2L)); + + assertThat(PermissionRecallDataset.dot(mixed, mixed)).isCloseTo(1.0, within(1e-5)); + assertThat(PermissionRecallDataset.dot(mixed, source)).isCloseTo(0.83, within(1e-5)); + } + + @Test + @DisplayName("섞을 Vector가 원본과 같은 방향이어도 무작위 직교 방향으로 유사도를 맞춘다") + void mix_fallsBackWhenOtherIsParallel() { + float[] source = PermissionRecallDataset.normalize(randomVector(new Random(3L))); + + float[] mixed = PermissionRecallDataset.mix(source, source.clone(), 0.9, new Random(4L)); + + assertThat(PermissionRecallDataset.dot(mixed, source)).isCloseTo(0.9, within(1e-5)); + } + + @Test + @DisplayName("확장은 실제 문서를 먼저 모두 넣고 목표 Chunk 수 이상이 될 때까지 같은 Seed로 같은 결과를 만든다") + void expand_isDeterministicAndReachesTarget() { + List documents = documents(5, 3); + + PermissionRecallDataset first = PermissionRecallDataset.expand(documents, 40, 0.7, 0.9, 7L); + PermissionRecallDataset second = PermissionRecallDataset.expand(documents, 40, 0.7, 0.9, 7L); + + assertThat(first.chunkCount()).isGreaterThanOrEqualTo(40); + assertThat(first.documents()).isEqualTo(second.documents()); + assertThat(first.documents().subList(0, 5)).noneMatch(DocumentSpec::synthetic); + DocumentSpec synthetic = first.documents().get(5); + assertThat(synthetic.mixDocument()).isNotEqualTo(synthetic.sourceDocument()); + assertThat(synthetic.similarity()).isBetween(0.7, 0.9); + assertThat(first.vector(synthetic, 1)).containsExactly(second.vector(synthetic, 1)); + } + + @Test + @DisplayName("목표가 실제 Chunk 수 이하면 실제 문서만 쓰고 Vector도 원본 그대로다") + void expand_usesOnlyRealDocumentsWhenTargetIsSmall() { + List documents = documents(4, 2); + + PermissionRecallDataset dataset = PermissionRecallDataset.expand(documents, 0, 0.7, 0.9, 7L); + + assertThat(dataset.documents()).hasSize(4).noneMatch(DocumentSpec::synthetic); + assertThat(dataset.vector(dataset.documents().get(2), 1)) + .containsExactly(documents.get(2).chunks().get(1).vector()); + } + + @Test + @DisplayName("허용 문서는 최소 1건이고 낮은 비율의 허용 문서가 높은 비율에 포함된다") + void permittedDocumentIndexes_areNestedAndAtLeastOne() { + List tenPercent = PermissionRecallDataset.permittedDocumentIndexes(1_000, 0.1, 545L); + List onePercent = PermissionRecallDataset.permittedDocumentIndexes(1_000, 0.01, 545L); + List tiny = PermissionRecallDataset.permittedDocumentIndexes(100, 0.001, 545L); + + assertThat(tenPercent).hasSize(100).containsAll(onePercent); + assertThat(onePercent).hasSize(10); + assertThat(tiny).hasSize(1); + } + + @Test + @DisplayName("다른 문서 최근접 유사도는 같은 문서 Chunk를 제외하고 계산한다") + void crossDocumentNearestSimilarities_excludesSameDocument() { + float[] base = {1F, 0F, 0F}; + List documents = List.of( + document("a", List.of(base, base.clone())), + document("b", List.of(new float[] {0F, 1F, 0F})) + ); + + double[] nearest = PermissionRecallDataset.crossDocumentNearestSimilarities(documents); + + assertThat(nearest).containsExactly(0.0, 0.0, 0.0); + } + + @Test + @DisplayName("Recall은 정답 집합 기준이고 정답이 없으면 계산하지 않는다") + void recall_isBasedOnExpectedSet() { + assertThat(PermissionRecallMetrics.recall(List.of(1L, 2L, 9L), List.of(1L, 2L, 3L, 4L))).isEqualTo(0.5); + assertThat(PermissionRecallMetrics.recall(List.of(1L), List.of())).isNull(); + assertThat(PermissionRecallMetrics.percentile(new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 0.95)) + .isEqualTo(10.0); + assertThat(PermissionRecallMetrics.mean(Arrays.asList(1.0, null, 0.0))).isEqualTo(0.5); + } + + private static List documents(int count, int chunksPerDocument) { + Random random = new Random(11L); + List documents = new ArrayList<>(); + for (int index = 0; index < count; index++) { + List vectors = new ArrayList<>(); + for (int chunk = 0; chunk < chunksPerDocument; chunk++) { + vectors.add(randomVector(random)); + } + documents.add(document("doc-" + index, vectors)); + } + return documents; + } + + private static RealDocument document(String name, List vectors) { + List chunks = new ArrayList<>(); + for (int index = 0; index < vectors.size(); index++) { + chunks.add(new RealChunk(index, name + " chunk " + index, 1, index * 10, index * 10 + 5, null, null, + "hash", null, vectors.get(index))); + } + return new RealDocument(name + ".md", "sha", name, "MD", chunks); + } + + private static float[] randomVector(Random random) { + float[] vector = new float[DIMENSION]; + for (int index = 0; index < DIMENSION; index++) { + vector[index] = (float) random.nextGaussian(); + } + return vector; + } +} From 9c4aa3c2f7d6b06c5529be6f28d72a9d810c277b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sat, 17 Oct 2026 18:57:43 +0900 Subject: [PATCH 3/4] =?UTF-8?q?docs:=20#545=20=EC=A7=88=EB=AC=B8=20?= =?UTF-8?q?=EC=84=B8=ED=8A=B8=EC=99=80=20=EA=B2=B0=EA=B3=BC=20=EC=B2=98?= =?UTF-8?q?=EB=A6=AC=20=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 예제 문서 15건·설계 문서 15건으로 만든 고정 질문 30개 - results.json을 CSV·Markdown 표·SVG 그래프로 바꾸는 스크립트 - 운영 findTopK SQL로 Custom/Generic Plan·HNSW 강제 계획을 비교하는 psql 스크립트 --- .../evidence/issue-545/explain_plans.py | 122 ++++++++ .../evidence/issue-545/make_report.py | 282 ++++++++++++++++++ .../evidence/issue-545/questions.json | 35 +++ 3 files changed, 439 insertions(+) create mode 100644 docs/test-results/evidence/issue-545/explain_plans.py create mode 100644 docs/test-results/evidence/issue-545/make_report.py create mode 100644 docs/test-results/evidence/issue-545/questions.json diff --git a/docs/test-results/evidence/issue-545/explain_plans.py b/docs/test-results/evidence/issue-545/explain_plans.py new file mode 100644 index 00000000..dd3b4b6e --- /dev/null +++ b/docs/test-results/evidence/issue-545/explain_plans.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""#545 실행 계획 증빙용 psql 스크립트를 만든다. 표준 라이브러리만 쓴다. + +운영 VectorSearchRepository.findTopK SQL을 그대로 PREPARE하고, 시험이 쓴 허용 문서 ID와 질문 Vector로 EXECUTE한다. +Custom Plan과 Generic Plan, HNSW 강제(enable_sort=off) 계획을 EXPLAIN ANALYZE로 비교한다. + +사용법 (시험 DB에 50,000 청크 데이터가 적재된 상태에서): + python3 explain_plans.py > plans.sql + docker exec -i <격리 DB 컨테이너> psql -U -d -q < plans.sql > plans-raw.txt + python3 explain_plans.py --mask plans-raw.txt > plans-psql.txt +""" +import json +import re +import sys + +SCHEMA = "docgrid_permission_recall_545" +QUESTION_ID = "D01" +CASES = [ + # (제목, 허용 비율, LIMIT(topK×4), SET LOCAL 목록) + ("허용 100% · topK 10 · 현재 계획 · Custom Plan", 1.0, 40, ["plan_cache_mode = force_custom_plan"]), + ("허용 100% · topK 10 · 현재 계획 · Generic Plan", 1.0, 40, ["plan_cache_mode = force_generic_plan"]), + ("허용 10% · topK 10 · 현재 계획 · Custom Plan", 0.1, 40, ["plan_cache_mode = force_custom_plan"]), + ("허용 10% · topK 10 · 현재 계획 · Generic Plan", 0.1, 40, ["plan_cache_mode = force_generic_plan"]), + ("허용 100% · topK 20 · HNSW 강제 ef40", 1.0, 80, + ["plan_cache_mode = force_custom_plan", "enable_sort = off"]), + ("허용 100% · topK 20 · HNSW 강제 iterative relaxed", 1.0, 80, + ["plan_cache_mode = force_custom_plan", "enable_sort = off", "hnsw.iterative_scan = relaxed_order"]), + ("허용 10% · topK 10 · HNSW 강제 ef40", 0.1, 40, + ["plan_cache_mode = force_custom_plan", "enable_sort = off"]), + ("허용 10% · topK 10 · HNSW 강제 iterative relaxed", 0.1, 40, + ["plan_cache_mode = force_custom_plan", "enable_sort = off", "hnsw.iterative_scan = relaxed_order"]), +] + + +def permitted_ids(plans, ratio): + """시험이 남긴 Exact 계획의 Index Cond에서 해당 비율의 허용 문서 ID를 읽는다.""" + for sample in plans: + if sample["scale"] == "50000" and sample["ratio"] == ratio and sample["method"] == "exact": + match = re.search(r"document_id = ANY \('\{([0-9,]+)\}'", sample["plan"]) + if match: + return [int(value) for value in match.group(1).split(",")] + raise SystemExit(f"허용 비율 {ratio}의 Exact 계획을 찾지 못했습니다.") + + +def build(backend, plans_path): + with open(f"{backend}/build/reports/permission-recall/question-vectors.json", encoding="utf-8") as handle: + vectors = json.load(handle)["vectors"] + with open(f"{backend}/../docs/test-results/evidence/issue-545/questions.json", encoding="utf-8") as handle: + question = next(q["question"] for q in json.load(handle)["questions"] if q["id"] == QUESTION_ID) + with open(plans_path, encoding="utf-8") as handle: + plans = json.load(handle) + source = open(f"{backend}/src/main/java/com/opensource/docgrid/domain/search/repository/VectorSearchRepository.java", + encoding="utf-8").read() + sql = re.search(r'@Query\(value = """(.*?)""", nativeQuery = true\)\s*List findTopK', + source, re.S).group(1) + vector = "[" + ",".join(repr(float(value)) for value in vectors[question]) + "]" + + print(f"SET search_path = {SCHEMA}, public;") + prepared = {} + for title, ratio, limit, settings in CASES: + if ratio not in prepared: + ids = permitted_ids(plans, ratio) + name = f"top_k_{len(prepared)}" + placeholders = ", ".join(f"${index + 3}" for index in range(len(ids))) + statement = (sql.replace(":queryVector", "$1").replace(":modelId", "$2") + .replace("(:permittedIds)", f"({placeholders})").replace(":topK", f"${len(ids) + 3}")) + types = "text, bigint, " + ", ".join(["bigint"] * len(ids)) + ", int" + print(f"PREPARE {name}({types}) AS {statement};") + prepared[ratio] = (name, ids) + name, ids = prepared[ratio] + print(f"\\echo '=== {title} (허용 문서 {len(ids)}건, LIMIT {limit})'") + print("BEGIN;") + for setting in settings: + print(f"SET LOCAL {setting};") + values = f"'{vector}', 1, " + ", ".join(map(str, ids)) + f", {limit}" + print(f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE {name}({values});") + print("ROLLBACK;") + + # 기본 plan_cache_mode(auto)에서 같은 Prepared Statement를 반복하면 몇 번째부터 Generic Plan으로 바뀌는지 본다. + name, ids = prepared[1.0] + values = f"'{vector}', 1, " + ", ".join(map(str, ids)) + ", 40" + print(f"DEALLOCATE {name};") + placeholders = ", ".join(f"${index + 3}" for index in range(len(ids))) + types = "text, bigint, " + ", ".join(["bigint"] * len(ids)) + ", int" + statement = (sql.replace(":queryVector", "$1").replace(":modelId", "$2") + .replace("(:permittedIds)", f"({placeholders})").replace(":topK", f"${len(ids) + 3}")) + print(f"PREPARE {name}({types}) AS {statement};") + print("SET plan_cache_mode = auto;") + for run in range(1, 9): + print(f"\\echo '=== auto 모드 반복 {run}회째 · 허용 100% · topK 10'") + print(f"EXPLAIN (ANALYZE, SUMMARY) EXECUTE {name}({values});") + + # 권고안 검증: 질문 Vector를 스칼라 서브쿼리(InitPlan)로 한 번만 계산하면 Generic Plan에서도 행마다 다시 계산하지 않는다. + rewritten = statement.replace("CAST($1 AS vector)", "(SELECT CAST($1 AS vector))") + print(f"PREPARE init_plan({types}) AS {rewritten};") + for mode in ("force_custom_plan", "force_generic_plan"): + print(f"\\echo '=== 권고안 검증 · 질문 Vector InitPlan · {mode} · 허용 100% · topK 10'") + print(f"SET plan_cache_mode = {mode};") + print(f"EXPLAIN (ANALYZE, SUMMARY) EXECUTE init_plan({values});") + print("RESET plan_cache_mode;") + + +def mask(path): + """질문 Vector와 허용 문서 ID 목록을 줄여 사람이 읽을 수 있게 한다.""" + text = open(path, encoding="utf-8").read() + text = re.sub(r"'\[[-0-9.,eE]+\]'(::vector)?", "'<질문 벡터 1024차원>'", text) + text = re.sub(r"'\{[0-9,]+\}'::bigint\[\]", "'<허용 문서 ID 목록>'::bigint[]", text) + text = re.sub(r"ARRAY\[\$3(, \$[0-9]+)+\]", "ARRAY[$3, ..., $n]", text) + # psql 표 머리글은 가리기 전 가장 긴 줄 폭으로 채워져 있으므로 구분선·빈 머리글을 지우고 줄 끝 공백을 없앤다. + lines = [line.rstrip() for line in text.splitlines()] + kept = [line for line in lines if line and not set(line) <= {"-", "+"} and line.strip() != "QUERY PLAN"] + print("\n".join(kept)) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--mask": + mask(sys.argv[2]) + elif len(sys.argv) == 3: + build(sys.argv[1], sys.argv[2]) + else: + print(__doc__) + sys.exit(2) diff --git a/docs/test-results/evidence/issue-545/make_report.py b/docs/test-results/evidence/issue-545/make_report.py new file mode 100644 index 00000000..4c3859e0 --- /dev/null +++ b/docs/test-results/evidence/issue-545/make_report.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""#545 권한 필터 × 벡터 검색 누락 시험 결과(results.json)를 CSV·Markdown 표로 바꾼다. + +표준 라이브러리만 쓴다. 사용법: + python3 make_report.py <출력 폴더> + +출력: + rows.csv.gz 질문 한 건 × 조합 한 개의 측정 행 (반복 측정 지연 표본 포함) + summary.csv 규모 × 허용 비율 × topK × 방식 조합별 요약 + tables.md 결과 문서에 옮겨 쓸 Markdown 표 + h1-final-results.svg 허용 비율별 최종 결과 수 (H1) + h2-sql-candidates.svg 허용 100%에서 SQL 반환 후보 수 (H2) +""" +import csv +import gzip +import json +import os +import statistics +import sys + +METHOD_ORDER = [ + "exact", "planner_default", "planner_default_generic", + "hnsw_ef40", "hnsw_ef100", "hnsw_ef200", "iterative_relaxed", "iterative_strict", +] +METHOD_NAMES = { + "exact": "Exact (인덱스 없음)", + "planner_default": "현재 계획 (Custom Plan)", + "planner_default_generic": "현재 계획 (Generic Plan)", + "hnsw_ef40": "HNSW 강제 ef40", + "hnsw_ef100": "HNSW 강제 ef100", + "hnsw_ef200": "HNSW 강제 ef200", + "iterative_relaxed": "HNSW 강제 iterative relaxed", + "iterative_strict": "HNSW 강제 iterative strict", +} + + +def ratio_label(ratio): + return f"{ratio * 100:g}%" + + +def fmt(value, digits=2): + if value is None: + return "-" + if isinstance(value, float): + return f"{value:.{digits}f}" + return str(value) + + +def write_rows(report, path): + columns = [ + "scale", "ratio", "permittedDocuments", "permittedChunks", "topK", "candidateLimit", "method", + "questionId", "questionSource", "sqlRows", "vectorFinal", "hybridFinal", "hybridLexicalRows", + "exactSqlRows", "exactVectorFinal", "exactHybridFinal", "sqlRecall", "vectorFinalRecall", + "hybridFinalRecall", "vectorMedianMillis", "hybridMedianMillis", "stable", "planUsesHnsw", + "planHnswActualRows", "planHnswRowsRemovedByFilter", "planResultRows", "planPlanningMillis", + "planExecutionMillis", "planSharedHitBlocks", "planSharedReadBlocks", "planNodeTypes", + "vectorMillisSamples", "hybridMillisSamples", + ] + with gzip.open(path, "wt", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(columns) + for row in report["rows"]: + plan = row["plan"] + writer.writerow([ + row["scale"], row["ratio"], row["permittedDocuments"], row["permittedChunks"], row["topK"], + row["candidateLimit"], row["method"], row["questionId"], row["questionSource"], row["sqlRows"], + row["vectorFinal"], row["hybridFinal"], row["hybridLexicalRows"], row["exactSqlRows"], + row["exactVectorFinal"], row["exactHybridFinal"], row["sqlRecall"], row["vectorFinalRecall"], + row["hybridFinalRecall"], round(statistics.median(row["vectorMillis"]), 3), + round(statistics.median(row["hybridMillis"]), 3), row["stable"], plan["usesHnsw"], + plan["hnswActualRows"], plan["hnswRowsRemovedByFilter"], plan["resultRows"], + plan["planningMillis"], plan["executionMillis"], plan["sharedHitBlocks"], + plan["sharedReadBlocks"], " > ".join(plan["nodeTypes"]), + ";".join(f"{value:.3f}" for value in row["vectorMillis"]), + ";".join(f"{value:.3f}" for value in row["hybridMillis"]), + ]) + + +def write_summary(report, path): + summaries = report["summaries"] + columns = list(summaries[0].keys()) + with open(path, "w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=columns) + writer.writeheader() + for summary in summaries: + writer.writerow(summary) + + +def summary_index(report): + return {(s["scale"], s["ratio"], s["topK"], s["method"]): s for s in report["summaries"]} + + +def tables(report): + index = summary_index(report) + lines = [] + scales = [scale["scale"] for scale in report["scales"]] + ratios = report["configuration"]["ratios"] + top_ks = report["configuration"]["topKs"] + methods = [m for m in METHOD_ORDER if any(s["method"] == m for s in report["summaries"])] + + for scale in scales: + info = next(s for s in report["scales"] if s["scale"] == scale) + lines.append(f"### 규모 {scale}: 문서 {info['documents']:,}건, 청크 {info['chunks']:,}개 " + f"(합성 비율 {info['syntheticChunkRatio'] * 100:.1f}%)") + lines.append("") + lines.append("| 허용 비율 | 허용 문서 | 허용 청크 |") + lines.append("|---:|---:|---:|") + for ratio in info["ratios"]: + lines.append(f"| {ratio_label(ratio['ratio'])} | {ratio['permittedDocuments']:,} | " + f"{ratio['permittedChunks']:,} |") + lines.append("") + + for top_k in top_ks: + lines.append(f"#### {scale} · topK {top_k} (SQL 요청 {top_k * report['searchSettings']['candidatePoolMultiplier']}개)") + lines.append("") + lines.append("| 허용 비율 | 방식 | HNSW 사용 | SQL 후보 평균 (최소~최대) | 벡터 최종 평균 | topK 미달 질문 " + "| 하이브리드 최종 평균 | SQL Recall | 벡터 최종 Recall (최소) | 하이브리드 최종 Recall | 벡터 p50 / p95 ms " + "| 하이브리드 p50 / p95 ms |") + lines.append("|---:|---|---:|---|---:|---:|---:|---:|---|---:|---|---|") + for ratio in ratios: + for method in methods: + s = index.get((scale, ratio, top_k, method)) + if s is None: + continue + lines.append( + f"| {ratio_label(ratio)} | {METHOD_NAMES[method]} | {s['hnswPlanQuestions']}/{s['questions']} | " + f"{s['sqlRowsMean']:.1f} ({s['sqlRowsMin']}~{s['sqlRowsMax']}) | {s['vectorFinalMean']:.1f} | " + f"{s['vectorShortQuestions']}/{s['questions']} | {s['hybridFinalMean']:.1f} | " + f"{fmt(s['sqlRecallMean'])} | {fmt(s['vectorFinalRecallMean'])} ({fmt(s['vectorFinalRecallMin'])}) | " + f"{fmt(s['hybridFinalRecallMean'])} | {s['vectorP50Millis']:.1f} / {s['vectorP95Millis']:.1f} | " + f"{s['hybridP50Millis']:.1f} / {s['hybridP95Millis']:.1f} |") + lines.append("") + return "\n".join(lines) + + +# 범주형 색은 고정 순서로만 쓴다(dataviz 기준 팔레트 1~4번, 밝은/어두운 화면 각각 검증). +SERIES_COLORS = [("#2a78d6", "#3987e5"), ("#eb6834", "#d95926"), ("#1baf7a", "#199e70"), ("#eda100", "#c98500")] +SVG_STYLE = """""" + + +def svg_style(count): + # 선(.l)은 stroke만, 표식(.d)은 fill만 계열 색을 쓴다. 표식 테두리는 .ring(화면 색)이 맡는다. + light = "".join(f".l{i}{{stroke:{c[0]}}}.d{i}{{fill:{c[0]}}}" for i, c in enumerate(SERIES_COLORS[:count])) + dark = "".join(f".l{i}{{stroke:{c[1]}}}.d{i}{{fill:{c[1]}}}" for i, c in enumerate(SERIES_COLORS[:count])) + return SVG_STYLE % (light, dark) + + +def text_width(text, size=12): + """한글은 글자 크기만큼, 그 밖은 0.58배로 어림한 글자 폭.""" + return sum(size if ord(char) > 0x1100 else size * 0.58 for char in str(text)) + + +def esc(text): + return str(text).replace("&", "&").replace("<", "<").replace(">", ">") + + +def line_panels_svg(title, subtitle, panels, series_names, x_labels, y_label, path, panel_width=280): + """패널마다 같은 x 범주에 여러 계열의 선을 그린다. panels = [(패널 제목, y 최댓값, 참조선 값|None, [[값...] 계열별])].""" + panel_height = 210 + left, top, gap = 56, 112, 36 + legend_width = 20 + sum(24 + text_width(name) + 22 for name in series_names) + width = int(max(left + len(panels) * panel_width + (len(panels) - 1) * gap + 24, legend_width, + 40 + text_width(title, 16), 40 + text_width(subtitle))) + height = top + panel_height + 64 + out = [f'', svg_style(len(series_names)), + f'', + f'{esc(title)}', + f'{esc(subtitle)}'] + # 범례: 색 표식 옆에 글자(글자는 계열 색을 쓰지 않는다) + legend_x = 20 + for index, name in enumerate(series_names): + out.append(f'' + (f'' + if index == 0 else f'')) + out.append(f'{esc(name)}') + legend_x += 24 + text_width(name) + 22 + for panel_index, (panel_title, y_max, reference, values) in enumerate(panels): + x0 = left + panel_index * (panel_width + gap) + plot_bottom = top + panel_height + step = panel_width / (len(x_labels) - 1) if len(x_labels) > 1 else 0 + out.append(f'{esc(panel_title)}') + ticks = nice_ticks(y_max) + for tick in ticks: + y = plot_bottom - tick / ticks[-1] * panel_height + out.append(f'') + # 패널마다 y 범위(topK)가 다르므로 눈금 글자를 패널마다 붙인다. + out.append(f'{tick:g}') + out.append(f'') + if reference is not None: + y = plot_bottom - reference / ticks[-1] * panel_height + out.append(f'topK {reference:g}') + for label_index, label in enumerate(x_labels): + x = x0 + label_index * step + # 양 끝 글자가 그림 밖으로 잘리지 않게 첫 글자는 왼쪽, 마지막 글자는 오른쪽에 맞춘다. + anchor = "start" if label_index == 0 else "end" if label_index == len(x_labels) - 1 else "middle" + out.append(f'{esc(label)}') + # 첫 계열(기준선)은 다른 계열과 겹쳐도 보이도록 맨 나중에, 속이 빈 큰 표식으로 그린다. + for series_index in list(range(1, len(values))) + [0]: + series_values = values[series_index] + points = [(x0 + i * step, plot_bottom - (v or 0) / ticks[-1] * panel_height) for i, v in enumerate(series_values)] + out.append(f'') + for (x, y), value, label in zip(points, series_values, x_labels): + tip = f'{esc(series_names[series_index])} · {esc(label)}: {value:.1f}' + if series_index == 0: + out.append(f'{tip}') + else: + out.append(f'{tip}') + out.append(f'{esc(y_label)}') + out.append("") + with open(path, "w", encoding="utf-8") as handle: + handle.write("\n".join(out) + "\n") + + +def nice_ticks(maximum): + for step in (1, 2, 5, 10, 20, 25, 50, 100): + if maximum / step <= 5: + top = step * -(-maximum // step) + return [step * i for i in range(int(top / step) + 1)] + return [0, maximum] + + +def charts(report, output): + index = summary_index(report) + scale = "50000" if any(s["scale"] == "50000" for s in report["scales"]) else report["scales"][-1]["scale"] + ratios = report["configuration"]["ratios"] + top_ks = report["configuration"]["topKs"] + multiplier = report["searchSettings"]["candidatePoolMultiplier"] + + # 그림 1 (H1): 허용 비율이 낮아질 때 사용자에게 돌아가는 최종 결과 수 + methods = ["exact", "hnsw_ef40", "hnsw_ef200", "iterative_relaxed"] + names = ["Exact = 현재 계획", "HNSW 강제 ef40", "HNSW 강제 ef200", "HNSW 강제 iterative relaxed"] + panels = [] + for top_k in top_ks: + values = [[index[(scale, ratio, top_k, method)]["vectorFinalMean"] for ratio in ratios] for method in methods] + panels.append((f"topK {top_k}", top_k, top_k, values)) + line_panels_svg( + f"허용 문서 비율별 벡터 검색 최종 결과 수 ({int(scale):,} 청크)", + "질문 30개 평균. HNSW 계열은 enable_sort=off로 HNSW를 쓰게 한 값이며, 현재 운영 계획은 Exact와 같은 결과를 낸다.", + panels, names, [ratio_label(r) for r in ratios], "y축: 앱 필터(최소 유사도·문서당 2개) 뒤 최종 결과 수 평균", + os.path.join(output, "h1-final-results.svg")) + + # 그림 2 (H2): 권한 필터가 없는(100%) 경우 SQL이 돌려준 후보 수 + methods = ["exact", "hnsw_ef40", "hnsw_ef100"] + names = ["Exact = 요청 수", "HNSW 강제 ef40", "HNSW 강제 ef100"] + values = [[index[(scale, 1.0, top_k, method)]["sqlRowsMean"] for top_k in top_ks] for method in methods] + line_panels_svg( + "허용 100%에서 SQL이 돌려준 후보 수", + f"{int(scale):,} 청크, 질문 30개 평균. SQL 요청 수는 topK × {multiplier}이다. HNSW 강제 ef100은 Exact와 같은 값이라 겹친다.", + [("topK별 SQL 반환 후보", max(top_ks) * multiplier, None, values)], names, + [f"topK {k} (요청 {k * multiplier})" for k in top_ks], "y축: SQL 반환 후보 수 평균", + os.path.join(output, "h2-sql-candidates.svg"), panel_width=480) + + +def main(): + if len(sys.argv) != 3: + print(__doc__) + sys.exit(2) + opener = gzip.open if sys.argv[1].endswith(".gz") else open + with opener(sys.argv[1], "rt", encoding="utf-8") as handle: + report = json.load(handle) + output = sys.argv[2] + os.makedirs(output, exist_ok=True) + write_rows(report, os.path.join(output, "rows.csv.gz")) + write_summary(report, os.path.join(output, "summary.csv")) + with open(os.path.join(output, "tables.md"), "w", encoding="utf-8") as handle: + handle.write(tables(report) + "\n") + charts(report, output) + + +if __name__ == "__main__": + main() diff --git a/docs/test-results/evidence/issue-545/questions.json b/docs/test-results/evidence/issue-545/questions.json new file mode 100644 index 00000000..f701588e --- /dev/null +++ b/docs/test-results/evidence/issue-545/questions.json @@ -0,0 +1,35 @@ +{ + "description": "#545 권한 필터 × 벡터 검색 누락 시험용 고정 질문 30개. 예제 문서 15건(파일당 1개)과 저장소 설계 문서 15건의 실제 내용으로 만들었다. 측정 전에 고정했고 결과를 본 뒤 바꾸지 않는다. Recall은 Exact 결과를 정답으로 계산하므로 정답 청크 표시는 두지 않는다.", + "questions": [ + {"id": "D01", "source": "examples/demo-corpus/01-remote-work-policy.pdf", "question": "재택근무는 한 주에 몇 번까지 할 수 있어?"}, + {"id": "D02", "source": "examples/demo-corpus/02-business-travel-expense.pdf", "question": "출장 숙박비 상한은 얼마야?"}, + {"id": "D03", "source": "examples/demo-corpus/03-annual-leave-guide.pdf", "question": "미사용 연차는 다음 해로 이월돼?"}, + {"id": "D04", "source": "examples/demo-corpus/04-information-security.pdf", "question": "USB 같은 이동식 저장매체를 써도 돼?"}, + {"id": "D05", "source": "examples/demo-corpus/05-it-asset-management.pdf", "question": "노트북 교체 주기는 몇 년이야?"}, + {"id": "D06", "source": "examples/demo-corpus/06-family-event-support.docx", "question": "결혼하면 경조휴가를 며칠 받아?"}, + {"id": "D07", "source": "examples/demo-corpus/07-training-support.docx", "question": "교육비는 1인당 1년에 얼마까지 지원돼?"}, + {"id": "D08", "source": "examples/demo-corpus/08-corporate-card.docx", "question": "법인카드는 얼마 이상이면 사전 승인을 받아야 해?"}, + {"id": "D09", "source": "examples/demo-corpus/09-onboarding-guide.docx", "question": "신규 입사자 수습 기간은 얼마나 돼?"}, + {"id": "D10", "source": "examples/demo-corpus/10-meeting-room-booking.docx", "question": "회의실 예약하고 안 가면 어떻게 돼?"}, + {"id": "D11", "source": "examples/demo-corpus/11-records-retention.hwpx", "question": "회계 문서 보존 기간은 몇 년이야?"}, + {"id": "D12", "source": "examples/demo-corpus/12-incident-response.hwpx", "question": "장애 등급별로 최초 공지는 언제까지 해야 해?"}, + {"id": "D13", "source": "examples/demo-corpus/13-flexible-work.hwpx", "question": "선택적 근로시간제의 코어타임은 언제야?"}, + {"id": "D14", "source": "examples/demo-corpus/14-personal-data-handling.hwpx", "question": "개인정보를 대량으로 다운로드하려면 누구 승인이 필요해?"}, + {"id": "D15", "source": "examples/demo-corpus/15-club-support.hwpx", "question": "동호회 활동비는 1인당 얼마 지원돼?"}, + {"id": "R01", "source": "docs/design/gimin-#131-hnsw-exact-scale-benchmark.md", "question": "HNSW ef_search가 40일 때 데이터가 많아지면 recall이 얼마나 떨어져?"}, + {"id": "R02", "source": "docs/design/gimin-#128-bge-m3-batch-size-performance.md", "question": "BGE-M3 배치 크기는 왜 그렇게 정했어?"}, + {"id": "R03", "source": "docs/design/gimin-#147-chunk-size-overlap-quality-benchmark.md", "question": "청크 크기와 overlap은 어떤 기준으로 골랐어?"}, + {"id": "R04", "source": "docs/design/barman-wal-follow-primary.md", "question": "OpenSQL 리더가 바뀌면 Barman WAL 수신은 어떻게 전환돼?"}, + {"id": "R05", "source": "docs/design/gcs-original-continuous-preservation.md", "question": "GCS 문서 원본을 별도 버킷에 보존하는 이유가 뭐야?"}, + {"id": "R06", "source": "docs/design/gimin-#517-reconciler-consistency-check.md", "question": "정합성 점검기는 어떤 불일치를 자동으로 복구해?"}, + {"id": "R07", "source": "docs/design/Gimini-3-#90-indexing-job-lease-recovery.md", "question": "인덱싱 Job lease가 만료되면 어떻게 복구돼?"}, + {"id": "R08", "source": "docs/design/gimin-#356-stomp-connect-authentication.md", "question": "STOMP 연결할 때 JWT 인증은 어떻게 검사해?"}, + {"id": "R09", "source": "docs/design/kangcheolung-#226-role-revoke-redis-live-permission.md", "question": "역할을 회수하면 권한이 Redis 캐시에 바로 반영돼?"}, + {"id": "R10", "source": "docs/design/kangcheolung-#456-hybrid-search.md", "question": "하이브리드 검색에서 희귀 단어 가중치는 어떻게 계산해?"}, + {"id": "R11", "source": "docs/design/gimin-#216-embedding-provider-load-protection.md", "question": "임베딩 서버에 요청이 몰리면 어떻게 보호해?"}, + {"id": "R12", "source": "docs/design/kangcheolung-#286-rag-processing-timeout-sweep.md", "question": "RAG 답변이 PROCESSING 상태로 멈추는 문제는 어떻게 고쳤어?"}, + {"id": "R13", "source": "docs/design/issue-505-hwp-hwpx-parsing.md", "question": "HWPX 표 안의 텍스트도 검색돼?"}, + {"id": "R14", "source": "docs/design/opensql-permission-replica-lag.md", "question": "OpenProxy에서 권한 조회가 복제 지연 때문에 틀릴 수 있어?"}, + {"id": "R15", "source": "docs/design/kangcheolung-#96-mcp-token-auth-infra.md", "question": "MCP API 키는 어떻게 발급하고 폐기해?"} + ] +} From 84474b2d5abba8472ec0cf3083bb821d5e981690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sat, 17 Oct 2026 18:57:43 +0900 Subject: [PATCH 4/4] =?UTF-8?q?docs:=20#545=20=EC=B8=A1=EC=A0=95=20?= =?UTF-8?q?=EA=B2=B0=EA=B3=BC=EC=99=80=20=EC=8B=A4=ED=96=89=20=EA=B3=84?= =?UTF-8?q?=ED=9A=8D=20=EC=A6=9D=EB=B9=99=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 현재 검색 SQL은 HNSW를 쓰지 않아(조인 9행 오추정) 권한 필터로 결과가 빠지지 않음 - HNSW를 쓰게 하면 H1·H2 재현, iterative relaxed가 대부분 회복 - Generic Plan에서 질문 Vector를 행마다 파싱해 15~21배 느려지는 문제와 해결안 검증 --- .../evidence/issue-545/h1-final-results.svg | 137 ++++++ .../evidence/issue-545/h2-sql-candidates.svg | 47 ++ .../evidence/issue-545/plans-psql.txt | 444 ++++++++++++++++++ .../evidence/issue-545/results.json.gz | Bin 0 -> 453596 bytes .../evidence/issue-545/rows.csv.gz | Bin 0 -> 249907 bytes .../evidence/issue-545/summary.csv | 193 ++++++++ .../test-results/evidence/issue-545/tables.md | 240 ++++++++++ ...in-#545-permission-filter-vector-recall.md | 386 +++++++++++++++ 8 files changed, 1447 insertions(+) create mode 100644 docs/test-results/evidence/issue-545/h1-final-results.svg create mode 100644 docs/test-results/evidence/issue-545/h2-sql-candidates.svg create mode 100644 docs/test-results/evidence/issue-545/plans-psql.txt create mode 100644 docs/test-results/evidence/issue-545/results.json.gz create mode 100644 docs/test-results/evidence/issue-545/rows.csv.gz create mode 100644 docs/test-results/evidence/issue-545/summary.csv create mode 100644 docs/test-results/evidence/issue-545/tables.md create mode 100644 docs/test-results/gimin-#545-permission-filter-vector-recall.md diff --git a/docs/test-results/evidence/issue-545/h1-final-results.svg b/docs/test-results/evidence/issue-545/h1-final-results.svg new file mode 100644 index 00000000..fb09e4ec --- /dev/null +++ b/docs/test-results/evidence/issue-545/h1-final-results.svg @@ -0,0 +1,137 @@ + + + +허용 문서 비율별 벡터 검색 최종 결과 수 (50,000 청크) +질문 30개 평균. HNSW 계열은 enable_sort=off로 HNSW를 쓰게 한 값이며, 현재 운영 계획은 Exact와 같은 결과를 낸다. + +Exact = 현재 계획 + +HNSW 강제 ef40 + +HNSW 강제 ef200 + +HNSW 강제 iterative relaxed +topK 5 + +0 + +1 + +2 + +3 + +4 + +5 + +topK 5 +100% +10% +1% +0.1% + +HNSW 강제 ef40 · 100%: 5.0 +HNSW 강제 ef40 · 10%: 1.7 +HNSW 강제 ef40 · 1%: 0.2 +HNSW 강제 ef40 · 0.1%: 0.0 + +HNSW 강제 ef200 · 100%: 5.0 +HNSW 강제 ef200 · 10%: 4.0 +HNSW 강제 ef200 · 1%: 0.6 +HNSW 강제 ef200 · 0.1%: 0.0 + +HNSW 강제 iterative relaxed · 100%: 5.0 +HNSW 강제 iterative relaxed · 10%: 5.0 +HNSW 강제 iterative relaxed · 1%: 4.9 +HNSW 강제 iterative relaxed · 0.1%: 5.0 + +Exact = 현재 계획 · 100%: 5.0 +Exact = 현재 계획 · 10%: 5.0 +Exact = 현재 계획 · 1%: 4.9 +Exact = 현재 계획 · 0.1%: 5.0 +topK 10 + +0 + +2 + +4 + +6 + +8 + +10 + +topK 10 +100% +10% +1% +0.1% + +HNSW 강제 ef40 · 100%: 9.8 +HNSW 강제 ef40 · 10%: 1.8 +HNSW 강제 ef40 · 1%: 0.2 +HNSW 강제 ef40 · 0.1%: 0.0 + +HNSW 강제 ef200 · 100%: 9.9 +HNSW 강제 ef200 · 10%: 6.0 +HNSW 강제 ef200 · 1%: 0.6 +HNSW 강제 ef200 · 0.1%: 0.0 + +HNSW 강제 iterative relaxed · 100%: 9.8 +HNSW 강제 iterative relaxed · 10%: 10.0 +HNSW 강제 iterative relaxed · 1%: 10.0 +HNSW 강제 iterative relaxed · 0.1%: 6.7 + +Exact = 현재 계획 · 100%: 9.9 +Exact = 현재 계획 · 10%: 10.0 +Exact = 현재 계획 · 1%: 10.0 +Exact = 현재 계획 · 0.1%: 7.9 +topK 20 + +0 + +5 + +10 + +15 + +20 + +topK 20 +100% +10% +1% +0.1% + +HNSW 강제 ef40 · 100%: 17.3 +HNSW 강제 ef40 · 10%: 1.8 +HNSW 강제 ef40 · 1%: 0.2 +HNSW 강제 ef40 · 0.1%: 0.0 + +HNSW 강제 ef200 · 100%: 19.0 +HNSW 강제 ef200 · 10%: 6.3 +HNSW 강제 ef200 · 1%: 0.6 +HNSW 강제 ef200 · 0.1%: 0.0 + +HNSW 강제 iterative relaxed · 100%: 19.0 +HNSW 강제 iterative relaxed · 10%: 20.0 +HNSW 강제 iterative relaxed · 1%: 20.0 +HNSW 강제 iterative relaxed · 0.1%: 6.7 + +Exact = 현재 계획 · 100%: 19.1 +Exact = 현재 계획 · 10%: 20.0 +Exact = 현재 계획 · 1%: 20.0 +Exact = 현재 계획 · 0.1%: 7.9 +y축: 앱 필터(최소 유사도·문서당 2개) 뒤 최종 결과 수 평균 + diff --git a/docs/test-results/evidence/issue-545/h2-sql-candidates.svg b/docs/test-results/evidence/issue-545/h2-sql-candidates.svg new file mode 100644 index 00000000..6cad1af2 --- /dev/null +++ b/docs/test-results/evidence/issue-545/h2-sql-candidates.svg @@ -0,0 +1,47 @@ + + + +허용 100%에서 SQL이 돌려준 후보 수 +50,000 청크, 질문 30개 평균. SQL 요청 수는 topK × 4이다. HNSW 강제 ef100은 Exact와 같은 값이라 겹친다. + +Exact = 요청 수 + +HNSW 강제 ef40 + +HNSW 강제 ef100 +topK별 SQL 반환 후보 + +0 + +20 + +40 + +60 + +80 + +topK 5 (요청 20) +topK 10 (요청 40) +topK 20 (요청 80) + +HNSW 강제 ef40 · topK 5 (요청 20): 20.0 +HNSW 강제 ef40 · topK 10 (요청 40): 40.0 +HNSW 강제 ef40 · topK 20 (요청 80): 40.0 + +HNSW 강제 ef100 · topK 5 (요청 20): 20.0 +HNSW 강제 ef100 · topK 10 (요청 40): 40.0 +HNSW 강제 ef100 · topK 20 (요청 80): 80.0 + +Exact = 요청 수 · topK 5 (요청 20): 20.0 +Exact = 요청 수 · topK 10 (요청 40): 40.0 +Exact = 요청 수 · topK 20 (요청 80): 80.0 +y축: SQL 반환 후보 수 평균 + diff --git a/docs/test-results/evidence/issue-545/plans-psql.txt b/docs/test-results/evidence/issue-545/plans-psql.txt new file mode 100644 index 00000000..c8befcbc --- /dev/null +++ b/docs/test-results/evidence/issue-545/plans-psql.txt @@ -0,0 +1,444 @@ +=== 허용 100% · topK 10 · 현재 계획 · Custom Plan (허용 문서 3559건, LIMIT 40) + Limit (cost=2083.56..2083.59 rows=9 width=1039) (actual time=338.748..338.754 rows=40 loops=1) + Buffers: shared hit=326857 read=41323 + -> Sort (cost=2083.56..2083.59 rows=9 width=1039) (actual time=338.747..338.752 rows=40 loops=1) + Sort Key: ((e.vector <=> '<질문 벡터 1024차원>')) + Sort Method: top-N heapsort Memory: 100kB + Buffers: shared hit=326857 read=41323 + -> Nested Loop (cost=254.45..2083.42 rows=9 width=1039) (actual time=1.693..331.536 rows=50008 loops=1) + Buffers: shared hit=326854 read=41323 + -> Hash Join (cost=254.16..2072.58 rows=9 width=117) (actual time=1.547..21.472 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + Buffers: shared hit=955 read=474 + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1651.20 rows=31908 width=50) (actual time=0.092..14.376 rows=50008 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + Buffers: shared hit=799 read=474 + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=1.429..1.430 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + Buffers: shared hit=156 + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.024..1.015 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + Buffers: shared hit=156 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=142915 read=7109 + Planning: + Buffers: shared hit=532 read=84 + Planning Time: 10.585 ms + Execution Time: 338.816 ms +(28 rows) +=== 허용 100% · topK 10 · 현재 계획 · Generic Plan (허용 문서 3559건, LIMIT 40) + Limit (cost=2085.68..2085.68 rows=1 width=1039) (actual time=2129.221..2129.231 rows=40 loops=1) + Buffers: shared hit=325849 read=42305 + -> Sort (cost=2085.68..2085.70 rows=9 width=1039) (actual time=2129.220..2129.226 rows=40 loops=1) + Sort Key: ((e.vector <=> ($1)::vector)) + Sort Method: top-N heapsort Memory: 100kB + Buffers: shared hit=325849 read=42305 + -> Nested Loop (cost=254.45..2085.53 rows=9 width=1039) (actual time=1.268..2122.201 rows=50008 loops=1) + Buffers: shared hit=325849 read=42305 + -> Hash Join (cost=254.16..2074.66 rows=9 width=117) (actual time=1.204..14.793 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + Buffers: shared hit=2 read=1427 + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1653.07 rows=31948 width=50) (actual time=0.097..8.108 rows=50008 loops=1) + Index Cond: (document_id = ANY (ARRAY[$3, ..., $n])) + Filter: ((embedding_model_id = $2) AND ((status)::text = 'ACTIVE'::text)) + Buffers: shared hit=2 read=1271 + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=1.098..1.098 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + Buffers: shared read=156 + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.015..0.781 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + Buffers: shared read=156 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=142896 read=7128 + Planning: + Buffers: shared hit=18 read=19 + Planning Time: 2.635 ms + Execution Time: 2129.413 ms +(28 rows) +=== 허용 10% · topK 10 · 현재 계획 · Custom Plan (허용 문서 356건, LIMIT 40) + Limit (cost=606.33..606.34 rows=1 width=1039) (actual time=16.706..16.710 rows=40 loops=1) + Buffers: shared hit=35199 read=3926 + -> Sort (cost=606.33..606.34 rows=1 width=1039) (actual time=16.705..16.708 rows=40 loops=1) + Sort Key: ((e.vector <=> '<질문 벡터 1024차원>')) + Sort Method: top-N heapsort Memory: 125kB + Buffers: shared hit=35199 read=3926 + -> Nested Loop (cost=254.45..606.32 rows=1 width=1039) (actual time=0.648..15.917 rows=5244 loops=1) + Buffers: shared hit=35199 read=3926 + -> Hash Join (cost=254.16..602.99 rows=1 width=117) (actual time=0.634..2.268 rows=5244 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + Buffers: shared hit=90 read=568 + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..321.89 rows=5186 width=50) (actual time=0.009..1.123 rows=5244 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + Buffers: shared hit=90 read=412 + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.624..0.624 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + Buffers: shared read=156 + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.007..0.442 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + Buffers: shared read=156 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..3.33 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=5244) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=15268 read=464 + Planning: + Buffers: shared hit=18 read=19 + Planning Time: 0.572 ms + Execution Time: 16.726 ms +(28 rows) +=== 허용 10% · topK 10 · 현재 계획 · Generic Plan (허용 문서 356건, LIMIT 40) + Limit (cost=601.82..601.82 rows=1 width=1039) (actual time=219.719..219.725 rows=40 loops=1) + Buffers: shared hit=39125 + -> Sort (cost=601.82..601.82 rows=1 width=1039) (actual time=219.718..219.722 rows=40 loops=1) + Sort Key: ((e.vector <=> ($1)::vector)) + Sort Method: top-N heapsort Memory: 125kB + Buffers: shared hit=39125 + -> Nested Loop (cost=254.45..601.81 rows=1 width=1039) (actual time=0.481..218.842 rows=5244 loops=1) + Buffers: shared hit=39125 + -> Hash Join (cost=254.16..598.45 rows=1 width=117) (actual time=0.434..1.957 rows=5244 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + Buffers: shared hit=658 + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..317.83 rows=5094 width=50) (actual time=0.010..0.953 rows=5244 loops=1) + Index Cond: (document_id = ANY (ARRAY[$3, ..., $n])) + Filter: ((embedding_model_id = $2) AND ((status)::text = 'ACTIVE'::text)) + Buffers: shared hit=502 + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.422..0.423 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + Buffers: shared hit=156 + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.003..0.249 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + Buffers: shared hit=156 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..3.35 rows=1 width=940) (actual time=0.000..0.000 rows=1 loops=5244) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=15732 + Planning: + Buffers: shared hit=37 + Planning Time: 0.436 ms + Execution Time: 219.750 ms +(28 rows) +=== 허용 100% · topK 20 · HNSW 강제 ef40 (허용 문서 3559건, LIMIT 80) + Limit (cost=2549.99..208973.32 rows=9 width=1039) (actual time=11.998..12.349 rows=40 loops=1) + Buffers: shared hit=358 read=324 + -> Nested Loop (cost=2549.99..208973.32 rows=9 width=1039) (actual time=4.593..4.942 rows=40 loops=1) + Buffers: shared hit=358 read=324 + -> Nested Loop (cost=2549.70..208962.48 rows=9 width=117) (actual time=4.571..4.719 rows=40 loops=1) + Buffers: shared hit=115 read=273 + -> Index Scan using embeddings_vector_idx on embeddings e (cost=2549.41..206540.12 rows=31908 width=50) (actual time=4.451..4.482 rows=40 loops=1) + Order By: (vector <=> '<질문 벡터 1024차원>') + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text) AND (document_id = ANY ('<허용 문서 ID 목록>'::bigint[]))) + Buffers: shared hit=92 read=266 + -> Memoize (cost=0.29..0.33 rows=1 width=91) (actual time=0.006..0.006 rows=1 loops=40) + Cache Key: e.document_id, e.document_version_id + Cache Mode: logical + Hits: 30 Misses: 10 Evictions: 0 Overflows: 0 Memory Usage: 2kB + Buffers: shared hit=23 read=7 + -> Index Scan using idx_documents_current_version_id on documents d (cost=0.28..0.32 rows=1 width=91) (actual time=0.011..0.011 rows=1 loops=10) + Index Cond: (current_version_id = e.document_version_id) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text) AND (id = e.document_id)) + Buffers: shared hit=23 read=7 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.002..0.002 rows=1 loops=40) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=112 read=8 + Planning: + Buffers: shared hit=37 + Planning Time: 2.843 ms + JIT: + Functions: 25 + Options: Inlining false, Optimization false, Expressions true, Deforming true + Timing: Generation 0.823 ms (Deform 0.291 ms), Inlining 0.000 ms, Optimization 0.525 ms, Emission 6.899 ms, Total 8.247 ms + Execution Time: 39.148 ms +(30 rows) +=== 허용 100% · topK 20 · HNSW 강제 iterative relaxed (허용 문서 3559건, LIMIT 80) + Limit (cost=2549.99..208973.32 rows=9 width=1039) (actual time=5.129..6.010 rows=80 loops=1) + Buffers: shared hit=1021 read=95 + -> Nested Loop (cost=2549.99..208973.32 rows=9 width=1039) (actual time=0.312..1.189 rows=80 loops=1) + Buffers: shared hit=1021 read=95 + -> Nested Loop (cost=2549.70..208962.48 rows=9 width=117) (actual time=0.300..0.924 rows=80 loops=1) + Buffers: shared hit=458 read=72 + -> Index Scan using embeddings_vector_idx on embeddings e (cost=2549.41..206540.12 rows=31908 width=50) (actual time=0.274..0.857 rows=80 loops=1) + Order By: (vector <=> '<질문 벡터 1024차원>') + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text) AND (document_id = ANY ('<허용 문서 ID 목록>'::bigint[]))) + Buffers: shared hit=426 read=71 + -> Memoize (cost=0.29..0.33 rows=1 width=91) (actual time=0.001..0.001 rows=1 loops=80) + Cache Key: e.document_id, e.document_version_id + Cache Mode: logical + Hits: 69 Misses: 11 Evictions: 0 Overflows: 0 Memory Usage: 3kB + Buffers: shared hit=32 read=1 + -> Index Scan using idx_documents_current_version_id on documents d (cost=0.28..0.32 rows=1 width=91) (actual time=0.002..0.002 rows=1 loops=11) + Index Cond: (current_version_id = e.document_version_id) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text) AND (id = e.document_id)) + Buffers: shared hit=32 read=1 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=80) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=239 read=1 + Planning: + Buffers: shared hit=37 + Planning Time: 3.156 ms + JIT: + Functions: 25 + Options: Inlining false, Optimization false, Expressions true, Deforming true + Timing: Generation 0.378 ms (Deform 0.190 ms), Inlining 0.000 ms, Optimization 0.248 ms, Emission 4.577 ms, Total 5.204 ms + Execution Time: 6.420 ms +(30 rows) +=== 허용 10% · topK 10 · HNSW 강제 ef40 (허용 문서 356건, LIMIT 40) + Limit (cost=2541.97..208868.07 rows=1 width=1039) (actual time=4.256..4.259 rows=1 loops=1) + Buffers: shared hit=362 + -> Nested Loop (cost=2541.97..208868.07 rows=1 width=1039) (actual time=0.164..0.167 rows=1 loops=1) + Buffers: shared hit=362 + -> Nested Loop (cost=2541.68..208864.73 rows=1 width=117) (actual time=0.157..0.160 rows=1 loops=1) + Buffers: shared hit=355 + -> Index Scan using embeddings_vector_idx on embeddings e (cost=2541.40..206532.11 rows=5186 width=50) (actual time=0.153..0.156 rows=1 loops=1) + Order By: (vector <=> '<질문 벡터 1024차원>') + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text) AND (document_id = ANY ('<허용 문서 ID 목록>'::bigint[]))) + Rows Removed by Filter: 39 + Buffers: shared hit=352 + -> Index Scan using idx_documents_current_version_id on documents d (cost=0.28..0.44 rows=1 width=91) (actual time=0.003..0.003 rows=1 loops=1) + Index Cond: (current_version_id = e.document_version_id) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text) AND (id = e.document_id)) + Buffers: shared hit=3 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..3.33 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=1) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=3 + Planning: + Buffers: shared hit=37 + Planning Time: 0.463 ms + JIT: + Functions: 19 + Options: Inlining false, Optimization false, Expressions true, Deforming true + Timing: Generation 0.293 ms (Deform 0.170 ms), Inlining 0.000 ms, Optimization 0.194 ms, Emission 3.902 ms, Total 4.389 ms + Execution Time: 4.569 ms +(26 rows) +=== 허용 10% · topK 10 · HNSW 강제 iterative relaxed (허용 문서 356건, LIMIT 40) + Limit (cost=2541.97..208868.07 rows=1 width=1039) (actual time=4.017..40.570 rows=40 loops=1) + Buffers: shared hit=2285 read=4192 + -> Nested Loop (cost=2541.97..208868.07 rows=1 width=1039) (actual time=0.148..36.698 rows=40 loops=1) + Buffers: shared hit=2285 read=4192 + -> Nested Loop (cost=2541.68..208864.73 rows=1 width=117) (actual time=0.142..36.502 rows=40 loops=1) + Buffers: shared hit=1988 read=4192 + -> Index Scan using embeddings_vector_idx on embeddings e (cost=2541.40..206532.11 rows=5186 width=50) (actual time=0.138..36.395 rows=40 loops=1) + Order By: (vector <=> '<질문 벡터 1024차원>') + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text) AND (document_id = ANY ('<허용 문서 ID 목록>'::bigint[]))) + Rows Removed by Filter: 775 + Buffers: shared hit=1873 read=4187 + -> Index Scan using idx_documents_current_version_id on documents d (cost=0.28..0.44 rows=1 width=91) (actual time=0.002..0.002 rows=1 loops=40) + Index Cond: (current_version_id = e.document_version_id) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text) AND (id = e.document_id)) + Buffers: shared hit=115 read=5 + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..3.33 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=40) + Index Cond: (id = e.chunk_id) + Buffers: shared hit=120 + Planning: + Buffers: shared hit=37 + Planning Time: 0.450 ms + JIT: + Functions: 19 + Options: Inlining false, Optimization false, Expressions true, Deforming true + Timing: Generation 0.290 ms (Deform 0.155 ms), Inlining 0.000 ms, Optimization 0.188 ms, Emission 3.685 ms, Total 4.163 ms + Execution Time: 40.913 ms +(26 rows) +=== auto 모드 반복 1회째 · 허용 100% · topK 10 + Limit (cost=2083.56..2083.59 rows=9 width=1039) (actual time=140.933..140.938 rows=40 loops=1) + -> Sort (cost=2083.56..2083.59 rows=9 width=1039) (actual time=140.932..140.936 rows=40 loops=1) + Sort Key: ((e.vector <=> '<질문 벡터 1024차원>')) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2083.42 rows=9 width=1039) (actual time=0.471..134.728 rows=50008 loops=1) + -> Hash Join (cost=254.16..2072.58 rows=9 width=117) (actual time=0.465..11.812 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1651.20 rows=31908 width=50) (actual time=0.027..6.802 rows=50008 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.435..0.435 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.004..0.260 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.000..0.000 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 3.004 ms + Execution Time: 140.956 ms +(18 rows) +=== auto 모드 반복 2회째 · 허용 100% · topK 10 + Limit (cost=2083.56..2083.59 rows=9 width=1039) (actual time=140.531..140.537 rows=40 loops=1) + -> Sort (cost=2083.56..2083.59 rows=9 width=1039) (actual time=140.531..140.535 rows=40 loops=1) + Sort Key: ((e.vector <=> '<질문 벡터 1024차원>')) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2083.42 rows=9 width=1039) (actual time=0.613..134.443 rows=50008 loops=1) + -> Hash Join (cost=254.16..2072.58 rows=9 width=117) (actual time=0.603..11.844 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1651.20 rows=31908 width=50) (actual time=0.025..6.725 rows=50008 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.576..0.577 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.006..0.400 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.000..0.000 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 2.758 ms + Execution Time: 140.553 ms +(18 rows) +=== auto 모드 반복 3회째 · 허용 100% · topK 10 + Limit (cost=2083.56..2083.59 rows=9 width=1039) (actual time=140.392..140.398 rows=40 loops=1) + -> Sort (cost=2083.56..2083.59 rows=9 width=1039) (actual time=140.392..140.395 rows=40 loops=1) + Sort Key: ((e.vector <=> '<질문 벡터 1024차원>')) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2083.42 rows=9 width=1039) (actual time=0.631..134.315 rows=50008 loops=1) + -> Hash Join (cost=254.16..2072.58 rows=9 width=117) (actual time=0.620..11.824 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1651.20 rows=31908 width=50) (actual time=0.025..6.720 rows=50008 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.594..0.594 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.007..0.420 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.000..0.000 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 2.896 ms + Execution Time: 140.413 ms +(18 rows) +=== auto 모드 반복 4회째 · 허용 100% · topK 10 + Limit (cost=2083.56..2083.59 rows=9 width=1039) (actual time=143.944..143.949 rows=40 loops=1) + -> Sort (cost=2083.56..2083.59 rows=9 width=1039) (actual time=143.943..143.947 rows=40 loops=1) + Sort Key: ((e.vector <=> '<질문 벡터 1024차원>')) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2083.42 rows=9 width=1039) (actual time=0.636..137.745 rows=50008 loops=1) + -> Hash Join (cost=254.16..2072.58 rows=9 width=117) (actual time=0.625..12.195 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1651.20 rows=31908 width=50) (actual time=0.026..6.897 rows=50008 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.597..0.597 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.006..0.412 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 2.902 ms + Execution Time: 143.969 ms +(18 rows) +=== auto 모드 반복 5회째 · 허용 100% · topK 10 + Limit (cost=2083.56..2083.59 rows=9 width=1039) (actual time=141.145..141.150 rows=40 loops=1) + -> Sort (cost=2083.56..2083.59 rows=9 width=1039) (actual time=141.144..141.148 rows=40 loops=1) + Sort Key: ((e.vector <=> '<질문 벡터 1024차원>')) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2083.42 rows=9 width=1039) (actual time=0.605..135.018 rows=50008 loops=1) + -> Hash Join (cost=254.16..2072.58 rows=9 width=117) (actual time=0.595..11.855 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1651.20 rows=31908 width=50) (actual time=0.024..6.727 rows=50008 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.569..0.570 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.006..0.399 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.000..0.000 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 2.825 ms + Execution Time: 141.165 ms +(18 rows) +=== auto 모드 반복 6회째 · 허용 100% · topK 10 + Limit (cost=2085.68..2085.68 rows=1 width=1039) (actual time=2114.771..2114.779 rows=40 loops=1) + -> Sort (cost=2085.68..2085.70 rows=9 width=1039) (actual time=2114.770..2114.776 rows=40 loops=1) + Sort Key: ((e.vector <=> ($1)::vector)) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2085.53 rows=9 width=1039) (actual time=0.871..2107.921 rows=50008 loops=1) + -> Hash Join (cost=254.16..2074.66 rows=9 width=117) (actual time=0.805..13.932 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1653.07 rows=31948 width=50) (actual time=0.055..7.709 rows=50008 loops=1) + Index Cond: (document_id = ANY (ARRAY[$3, ..., $n])) + Filter: ((embedding_model_id = $2) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.748..0.749 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.007..0.416 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 2.056 ms + Execution Time: 2114.887 ms +(18 rows) +=== auto 모드 반복 7회째 · 허용 100% · topK 10 + Limit (cost=2085.68..2085.68 rows=1 width=1039) (actual time=2098.872..2098.879 rows=40 loops=1) + -> Sort (cost=2085.68..2085.70 rows=9 width=1039) (actual time=2098.871..2098.877 rows=40 loops=1) + Sort Key: ((e.vector <=> ($1)::vector)) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2085.53 rows=9 width=1039) (actual time=0.754..2092.132 rows=50008 loops=1) + -> Hash Join (cost=254.16..2074.66 rows=9 width=117) (actual time=0.691..13.377 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1653.07 rows=31948 width=50) (actual time=0.064..7.428 rows=50008 loops=1) + Index Cond: (document_id = ANY (ARRAY[$3, ..., $n])) + Filter: ((embedding_model_id = $2) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.624..0.625 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.009..0.449 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 1.362 ms + Execution Time: 2099.007 ms +(18 rows) +=== auto 모드 반복 8회째 · 허용 100% · topK 10 + Limit (cost=2085.68..2085.68 rows=1 width=1039) (actual time=2101.027..2101.041 rows=40 loops=1) + -> Sort (cost=2085.68..2085.70 rows=9 width=1039) (actual time=2101.027..2101.039 rows=40 loops=1) + Sort Key: ((e.vector <=> ($1)::vector)) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2085.53 rows=9 width=1039) (actual time=0.722..2094.283 rows=50008 loops=1) + -> Hash Join (cost=254.16..2074.66 rows=9 width=117) (actual time=0.669..13.312 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1653.07 rows=31948 width=50) (actual time=0.062..7.448 rows=50008 loops=1) + Index Cond: (document_id = ANY (ARRAY[$3, ..., $n])) + Filter: ((embedding_model_id = $2) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.605..0.606 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.009..0.438 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 1.184 ms + Execution Time: 2101.166 ms +(18 rows) +=== 권고안 검증 · 질문 Vector InitPlan · force_custom_plan · 허용 100% · topK 10 + Limit (cost=2083.57..2083.60 rows=9 width=1039) (actual time=142.463..142.468 rows=40 loops=1) + InitPlan 1 + -> Result (cost=0.00..0.01 rows=1 width=32) (actual time=0.006..0.006 rows=1 loops=1) + -> Sort (cost=2083.56..2083.59 rows=9 width=1039) (actual time=142.462..142.466 rows=40 loops=1) + Sort Key: ((e.vector <=> (InitPlan 1).col1)) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2083.42 rows=9 width=1039) (actual time=0.693..136.240 rows=50008 loops=1) + -> Hash Join (cost=254.16..2072.58 rows=9 width=117) (actual time=0.653..11.884 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1651.20 rows=31908 width=50) (actual time=0.028..6.682 rows=50008 loops=1) + Index Cond: (document_id = ANY ('<허용 문서 ID 목록>'::bigint[])) + Filter: ((embedding_model_id = '1'::bigint) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.623..0.623 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.007..0.429 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 3.107 ms + Execution Time: 142.486 ms +(20 rows) +=== 권고안 검증 · 질문 Vector InitPlan · force_generic_plan · 허용 100% · topK 10 + Limit (cost=2085.65..2085.65 rows=1 width=1039) (actual time=162.628..162.634 rows=40 loops=1) + InitPlan 1 + -> Result (cost=0.00..0.01 rows=1 width=32) (actual time=0.050..0.050 rows=1 loops=1) + -> Sort (cost=2085.63..2085.66 rows=9 width=1039) (actual time=162.627..162.631 rows=40 loops=1) + Sort Key: ((e.vector <=> (InitPlan 1).col1)) + Sort Method: top-N heapsort Memory: 100kB + -> Nested Loop (cost=254.45..2085.49 rows=9 width=1039) (actual time=0.863..155.460 rows=50008 loops=1) + -> Hash Join (cost=254.16..2074.66 rows=9 width=117) (actual time=0.800..14.465 rows=50008 loops=1) + Hash Cond: ((e.document_id = d.id) AND (e.document_version_id = d.current_version_id)) + -> Index Scan using idx_embeddings_document_id on embeddings e (cost=0.29..1653.07 rows=31948 width=50) (actual time=0.078..8.206 rows=50008 loops=1) + Index Cond: (document_id = ANY (ARRAY[$3, ..., $n])) + Filter: ((embedding_model_id = $2) AND ((status)::text = 'ACTIVE'::text)) + -> Hash (cost=200.49..200.49 rows=3559 width=91) (actual time=0.713..0.713 rows=3559 loops=1) + Buckets: 4096 Batches: 1 Memory Usage: 473kB + -> Seq Scan on documents d (cost=0.00..200.49 rows=3559 width=91) (actual time=0.009..0.422 rows=3559 loops=1) + Filter: ((deleted_at IS NULL) AND ((status)::text = 'INDEXED'::text)) + -> Index Scan using document_chunks_pkey on document_chunks dc (cost=0.29..1.20 rows=1 width=940) (actual time=0.001..0.001 rows=1 loops=50008) + Index Cond: (id = e.chunk_id) + Planning Time: 2.133 ms + Execution Time: 162.764 ms +(20 rows) diff --git a/docs/test-results/evidence/issue-545/results.json.gz b/docs/test-results/evidence/issue-545/results.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..2f31e7b32ecc30c94056af5fb3b5798508efdf32 GIT binary patch literal 453596 zcmV)SK(fCdiwFoUHq&YX19D|^b!>EVE^2dcZUF4PU3Vhcu`T+3enpM;%Q@qAu3oXe z*M4i;y=~uX`}lTye~i1w88r~fs=|dxH6T^h-edpw8?h23KoCg^NRhD0?QTjSgdpc! z5iuiX{MY|@_wKv9axi-zNg3E zSF`!YX*v7XaylDTlk`pCMR@l=e_!5=29tO6mjcB7$J>vf{xA4ni*Nh*SPtjaboJY$ z2m;oh^_Ox!A5A{a=>Gr0JMgYgli3$het17C2h-uF^o>TQ?^@nDnwQhTeDqkppA84~ zQ&%58tUcsz@HO4}e*SPjE@z8h75}U~Dm~(^ynBE5FrJU@$D?vu|9;Cy%{~qAt$Kg+ z@Zkgg9`$`H{N=aH`@dAv&-4WKt9<_Y-C#7Cmy^L{SiWC-j9Yn7iBCFAzs7Jd{Di;E z=s(N)iT?2PnCW10Tiw0CAAH0|;d@r!|NLXV;$73hcwCOt&%qb@{=S^P|2UX`TKq0* z>w`C6;PSV7di4Dm-l!@FtL6Rw7}c)@KU)04f2R+%n*F%^#0K=)x%!2E z_m9EjVB_+U0>a|X+neF1-KU4i=k#@Ly-Ih?K9BD2%iCY7hv~4sM*id7 z|5^E#pO!!OmvUH5Zqp+S;HyV;fVIx~`oYs`_VkngidStpoBuMp8;u9k(fr%Wl|8H; zCiB%-K%dGe{lZdM&`LQMgOzAJc$ohU4f|S-usg;c`KHYB@_j`-5jUbJ;%6J9K z`{UyNpsm)LeERli7k4{prPbaV2p*s5`J$=s#~&FrI3JCUQp>o!zZ@)Hzjsm@ z6Y-5PE=1*?u8JDRTY!$g`uf`>g?|3A_2>1sa=LtH(?`0vA8~s-x*g2Rzg5-vFKgg6 zOMrj7nT~FMnhb8nyXzyImq z-#r0h_P%D>_oLgd@Ba5+-`!TjkN5+L1MWsMLeu+ci9dS$9xo~J{B7_4<(EJG^$+jp zhrRos^;5iCJQK3aXi~nf?q~mNiLA+o(Z>g3-1@aFKE7#*BtF7#`Sx>F)nxEtG%o+? zhaZ0Uml;C&hmZKSPCk$Be)!XEIr!VO`Un1i*$<1C_=6aZ2M@R955zw6X<5$j7XIGT z4-ezwelVMt57WX9jJ&=5;bt)WTu$&?#1BjQSzwy@>2Jt&UHtT$htc@<2l}n2_(ad| za8bX*c+X&5)UQkYV}@_l;-&ujfP8?Dn|1X<@omEIp8YV#pNlVUhF99`hw^cBTO!;4 zfG_%?Ebwyp;pv5KeYt1#-Y#EMQD3C%uef@c-#^UH=Q2od_}VL^!}Ncj)vKgewEh|~ z78)-P!SCo={_gvC-`Ny0)rVq=M({&eJO+gPJ~18sOV*U@O;jtsCoJIclASlzx}S+w+Tjz^U-ka{^g4qzKEL^&-S}* zcRW2;y7ylRX$h;YI$^z=!pF- zfn?=j80Dy7Ra8&m&77i#-+r6d?>xS;V7*&=isgOyT^<48p05A-VEl9^Mpl4fmD&QM z`m z1_O=*mvX`@>%7-mbLf&=;bmlWUV9a`;tP@IwKX9e31H73_x39(eR_vLOk2l3KW0*g zfsB26b12LV`|N8lTO`!8hr2txy7oH3UVczbJh+PmSAvoBwr*yRPbaUEYoSScN#)nU zaQ-5QuPJbeo?pj5S6^m-StP+~<(OOCGn%YD{qoCoR{iwlI;#EW(PS`QK9> z5B=p6#&!Si>T6PK8RuS4RX=q1Q>8cHU-y4$>G6Nu|L4=2k)qM%17Yp$tM3|Z+_Uml z?fT}I57~T@l{abEle~P$_E+*ZEB`VYk4H6ySz)!bMy)?2+Lea{;iZ1&!20K`JxdHi zTQD}PKV8BjynlSl(k?Uq`W^C?{O)b5U!Ketk z!KppHAK^K5--1pZwZPL5oZpR3iwQ`;jOjX#O#0>Cmmi*m(|!ZewF0DlD?nn(Z0!z6 z%4SH?w=?hAOK{0%gyM6C28c=yqE<-tq=#*RCjwX-YhX7x1HOIuhdntJi0C7GaOw+# zCdiD;cW-R{!{8KrYR5YCHuMF#6Q<@U zil^CyO|u=FhQTIme&jDPBzMCxiogVG?c?5+SF;O&yME*aYA8WS>A|_(fKv!3?F@Kc zL$Tg%QssuQ>4b@@0dg0ExI>i9jJ@l?o)_ME*MvQsg$L!TdDr?IDmnjS}fiT=~?dsxj(9o-X#^6-D!5NMsJceC}4BHVI z7(}Xr5DB4WFxv!4)eOmvf3ly(XY&NBl zpo9z|a_u)dXknjQ0k?{eh(Z$^bdHNb;TedJfg1X^2o<|f2pkMCuUJEbYJlNWX}=R3 zQWZaNl~un-dg#|jMs#C(2(~#t^u~ES!+ccf-}&p}DI2Qs~GcYee%4{(lZ07TsuA0fv4ws zq_D@p)jm;f2y0R9;m4=896$}es1PI`uMMHE5GgZ2Ibrg^ z#51UteRg|6QXd(TPKMAB(PHKpfsbg7GyJiMuN>ma>pd|jp9NXouX=}Db+1ie0~4*a z99LGYCPW&GL%Uwv?s;W(E%PFRqE}9_w8BfvHoYQJy>rh_vfFUz1*V>{E6L<7QFuNE zuo;j2!=n+_D(6_Y0PP&tKY(wRdYNu+c*x?t)<5G!@K_}Y;Ot%}EdSj>FmhXNu zEJ~qZySno#za3-L+MlBs?ra*%wRGG85g7vlgPA9={daVtDAgAQ)3$16 z6lyCn;T0z)G&LFk2gTspos#zLkZJ6m!uwwhU8D4=--+)fTMX*ihB>*?t#?!AQHVsmWVrQS7uv?naH?SGQOCP(9#M4B(;$wHg*gPk@*U*4b&0@6O z`KjIxvC#ql$fw?ez1o6WHXrN*S=$n_mV>NzmYwdh2wBg%{2*k5wXA^Rg^aGd;jZ&Q z?3+8k9t2p~%1`7i0UKi2FW}XC#Jlbo2n{Zp)jx+I)C%n#Tgy!gNqB9=Qtzzfv=zMa zY+QXvuONCwwy$0sjKw`hpuKPxTZhFH4NTHH0s`az2tZq1`N>BwjEbEt z&p8KhwpIBqHDR^3Qm1;8nb(TL+w$=G%cmVePsND@JBtV z7C5EBA$a)SH!wotX%GzGRbyp$HUV#n!AtR9ee=W+gRW>`uRt~Kqxv=D&F!J{#g;l> zw;dd-Bo?t_HT1)9O^xB==L&KwMX&cuxo0b zJw(pKH=Li80vcS7thb{RL0NW%dD@BWqU~pI={0$&D$(Q}2k{7#!_PnF5~rPFwsdM*aLO*uKy2x?l~mV* zt-f2KW{e%K>RI7vXB0OWSX(F>MmaEE;Nj#rm{}*3_Ual?cDb^i5lv2t>coJf(IK-g z7aXGNfokiWxR04d#@ zs2i^K7-*2NPfbNdXeq<*VGLzsh*Cxsd{oxOgErP^Oy#WR%oH(W zlwgBlXBv#r4KoeO$-a*7c4KjRM6r+T_ZUg59Z16&9HB}vo15$@1h&$N!6?;wJ)1dth`c9MTq~$vGzu@q&h+hbMpD-pYyKGa^8DUI zUWohXd|`LQu=P1veIY2x?gt0VMZs!T>GQ(tm4tL##fDB`=rMQ&BMxbln9)%8+ zdFiA55|LrA*fg|_P3=CWrQR8u9t{LY!#L=R&33uqB4zBw%cFLE^uCQRh4!m8a4nm1S8fP*a{srmjqHf ztap4n_C=uVd<;o5-p?`3`aBS&UNBe zOU2F{1!{g_7sq&^ls8;I8OVCHNnJehUZ)#AyMpYc6z&GvroUFZx+ZMMjE%M}jb+Xw zLXAQtx&Fr-A{)xEUI7h9F%3P?Gt@-uxwWNA;Q7VM_*w+3SYrt*yy|qq;jO{gRwtCg zw?Jdy(755HsSg6DJuftBdyr}m>sW@Pz|X6-{~ZQKuQclvs>wJhaI!suQG3<}-;xhZ z!~Wol0%L0paI>YR4K^c(&FGHSQv;iemGq&L^T&m%3;TNd?g8A5_w6X>rnhGGf9|ku z8;F`it>5B*zcFeEDsbYxODmACpg4L?{-HUAJkJy|kh}S_lD9)LjE0x$F}!iy3N;a< z?Q%Bo3?TRB2B1eooYL+5DomH>V;cY2=Xl&}c(<*~cG$rE7!wR< zvulL|$(~ZQgwS`mgz#)&^(sv;!|P(;lLz|*23>$EVHIu z?T0hFaTfZ6P`6`!{aYP{G1f<*iS?1VTi5gJ;q4wOpKEt62EC^l?@2U2z+}?8=QD0Q zoUQV#2Mt~;QnPDc5Y1Ff?PpMXa@){+K3?6|GdtQ--j5$Wmxm9@U>1{R)0z^+a~SZoBC8z&WpqL!8U zq=2A&F9-ZOaR$vY-<8UIv@4o*<`v5FyP~zA&S!A!MBmEl!3)m0vdEW3zGI4fu(HK= zD>fm9hEXAKz)suZXX2vvSA@qMkF#u-wQ>TCbF87D5y(I?Fvp;#DJ!Y23tU<0%SvC@N*~6o#hQAFC8lA&8TE1NE^R{J3~Gl+ff;mH13Hk7MVwU)zDyp|MRwJhh{n1O z#rJbT+^e+0jJzvBo)*pkzsXF|25zom>a28J83t2rr=APao8 z_^5WMR)@oN#}i)&T$>)+2CbUYZO;O&E~sY@?X44stR2#~`nS>fS}!{rp80KOC&d7NfzXTLnJ-v|v`^ z=nF>bVOnI)Jg+44jKapqE~+4x(bZw#9c#2@w2g)}Mqb4sd5}^WcQ|ma!!TwPZVHwA zos@AOWwK=ky2rA@$h~{5oko4|>^uiTR{6+wzb5Zs&z+TsSm|AZI&!v%J-O3pJ|VA1 z6(Pqc#rLyXc9oM(&cBj%tpV^2dwVYgYgvoTsB7+^6Tw)`uZhu3Y9yXtp7(K++VoTs zc>|l$48AJ?pTQ$9*q}L&U=uA{iZR0Z6##GT3kQf@&NjXs`m!RK70DB7942_J+1*U| zmT*U<5D_BQaFNR4X9B`*xCS&+YRWJpmxCGwFIGOSgVo{1+tW^;PJK{w`guK?A&-U2iLqI8Z zkD7_nbvZhD4#3NbWL6|kphz0IvXXH7(wnI53rCNQCW7FA-46^}xafTBXA{B6jZpIe zc|DS(T`^?b8kt53hTC#O&2+1A=QedDkZq*yV|}(mr__wPt3jQ#zCn_-Rw-7Jw8n9W z)9N}Am-WbuJPvt&b=l0;0Z9e$!YOC71;;nE;HVIL+6~Z=*=p7!vmUvhIV+nDk{&5p z>nnx%>C66tjk>9gJ+0n!WzU-R$gD^1W7irRoOUG01}kOB4oDi|qg*+iuK=uBkj#SQ z@dZh3SBwR=mwjk=9NUQ=j4|BmI7>fe#aQc-!I}lhoPX{mNJfb42nc}iF22}Hydg*` z?O5o67RuP`2x_w=nI*{+N|MCQPD#$E2NVL*uz)pAvReocurU~Q3PI9J`5Zz%LH3!Z zcK9#bCkonvS$fXUv1O~;bx5imd8g#OmYd+opwn)g0lU9>7aDCN8ts;o?qZ6tG77o6 zuWFl_jt#n9>5g=61@dKz*s&u*3b@ zD&eeF-0&kr>BzRjUn#;~B{^oKUCM}~3K(#LW3xKrqo$P%Y@?~gM+qG)>*t8aoe|oa zD}mbU^u-LWr%Xt}F5RYeSu?`%1&FAQxG&C7UF3Nw#qsHXLiu z(T3!a=mByY@Tj2IIf)h4q;IRogP$~Ni$dh}Vnq4q7+hJ%J819nncYzX^rQz~mRPK5; zyD+-Z>T_YL11LP44BG^%93F8}z3}*6$FD-<`Gy%@WFetxnU(KtF|^5k!e7SWRp8Jl<%Fimc_s)o7^MoYG&^5! zjWNPH-U8JD=7kcV&uuQ9SfL&lexcp0VQ=6^DPDwQjV?uL!?DL9qB`Q_68Rj$^GNTu zQ&W;Njbe2Sj9GQGk~35UuTv&|iq7owhG|>f@W{vH#T<+zOql(UxjE;)Lse9^5szUH zJbJnM2M&@pXn@|ZDt?d+5iB5LHLzzU_3Q|1;Ux2f=Z!{{(ReI0KCcOUH3HX8T6YvQ zhL)!zheix6wPZ~*lork@$4VQBMPZC%{f5D5EuQVDE+3AENsT#{&9Us3ST<_wxW;cf zK9%A3Faq_DOK3o=cjRPsg=l?+5}Co8!MY{Tl??{PiVpMe!d`J5tP=2T<99H+;DwK- z&xz;BZBN54h=%PD4Gcsf3TQy3-Hl3hI4W0Oz%C%F%hn=k-7v5xp^8>0@VuL_#&^;x z&DV88ltS620p3f3*CASD2HHgd4MRn3jOEIw7%93S*$o$YpVhrmPCW7HGgC*Dyz0v94 zN6|Cf5!FQuAftI+rM4|x#X}GIgf(G9vLVsa@l_Y<3D9s76U6fdsLB969zc(vF8+8L5^n8dfp1jCK)BPOGb4XOpEqBbG{85Wa{EEcb)5PF!%n1RUv)7Wjt z(o2xqy&~DDyu{p2OT0X(;CPggW2=vc(jbsM*s&_=x>5YvVLUmF@R$NvcFs|Q0I7q& zMx4GvUd%A%Bt^#@p!;Q+QEKJdU~?ZS5mxiy7!OK0gRm-Y&SI`(Qwaru$0mRjo(^Kx zj6-?hWtUy7Qa}SX9Rp2eO!g|x?ryIVH}I$e!_M{VI^|`1!>ty^0GBE%L?`%oik)re z2&rsRn!!cj3PuHOSa9K`#zG4OX0<%FC@qxUcP7rC6}GZH=~dg4Izy97tQcXv6ej8y z4s2_3VQ>OeRNUUfATiUGY9+C|Xq>&$n{-o;YqOsTm}P1pnl>^izZ++IW4%+;#)&8W z`9yX#px5nc!XbU2r&yh4bv#<~7KAi0vTN_D`WaaJU@%lciu_3^sX!WRcOw>O0nxT< zp%>6*!69?@X(3xJO#>F9v6Ye!a9W6JyH@>_2klIA=eEu;;JYmqHxrCerL>;)HZ>qZ zaBQVhkrS1Y4T;~yL9(qr2x@b$;gJw(Hfo)^38Bn%B8FNGm<*?dq=Q2+h#AthBkizG znsuu<)d)$&u43}Ra)XFq@R)QB{oG`r2$EX~tbmp{#tuyu-xibE@$^U@i`xtOFTv>d z5geEvz-_=WClCW214boeRD95C1;C1`zZH#9gte4$ThJN=M!xpiu$b*m4}fBjVOO-! z{#*=ivcT7{;B!&3gM;L}A0@v)&1h<2crmxXu1iEAVRdZ3Z}97txhcp)P~Pc7F1oYc(~f+4Kz>kf+pXp+{TSZP;(i{2nPB__o{A_Q>e8J*wUyePDND z)4DaQ_+@HZ(Ub)1n(yh@v{#bNy;=ZNTQ8Tr#0JaBfrp^dHKP(vjY0NvcG~uH$J3Df!nKS`v6l1uGkq{ z>GX=RRP#bLDf8~sFx58bqaB{sGj4OQM5)s)S0Mtc3p(RNgD{BkX~WWsAhfdl#C@cJ zd$3o7)AKPX?Rzds>LBcQa_%$H8&0te*c@?I(2`w%VmCo)vP^60?e>A)#I5iLAqILp zi_}*W(*6U|`}I?DCm{Foz3}SzO{U(rOfWTXN)v!B1(A<;Xs|;x)IOt?WLw&lP^!z> z-D6|2S5~9yARr!LKvy*b@vIfOa@zXh_jwl#< zVIr_a&me*gz;0#IN`PWV1_4p3w7r^K?xGY!FXG$7GRh z1OAEya7z(CGDTD+ECNmjZi@^CtH!V+O%_COQX1fksNj9H@#S$Dp%Oo~(9Wag z)EM<~x2>q%1C-7~yBS}-j|WD}U{*8dq|(AD&)XjYsg%4kjU~Gr&+Xz?8YwEq6nwHx`pMT=z6<@8eQM3NqUO3laf=|XZ;dy{@ z8`zwXI`ADcmx8o&F&fR7J6p^t%WnzwLPx9E11?g%*g19-Q3R>45OHsAKbnzuu2Lhp zbQsn_#uzFev)Hprcxk!V(t2S@(ht{wzSoJ2WOw!YLaEseKFoaG@l-}7?4bitM+6{! z&ftzW4JD0Q|8o@0QER{Ur0x)P!wE-~6W}yQu$K46z{65W&jzR-VILy+t4Ic4sW|R* z6xypXKysKhTDRK&!eiE*7iP@{W)IUNj{?#4O=^ctEFyHmi*-{Q5YJRiT(OHD3Ge>}hW#3S058mN8Vm}w{%gmIE<0I>8Vfz$tkq6Mw4 z%jv|kz~vizmmU?CfoVM1aujz7(xCV3+A;eKFv{aw?Uli4Y-zLsTo;{g)eg|q>ALHs?Ly|RpMQCkS}zo9yC3m>#npTi zK0O=01_5$aK?08H@eIqSI_g4(*DP)*8j)(M!b+@oO)fx4l(OdhVQ>q1i`4 zQwt>{>#$)0zI%~7DyQ(EZO2fUmJKj@mcMOqNz8Ztx@0%Zv{`h3gj_e4r9F@spgV-5CvA)SF;z6sQw;*45`gz%ErxyC00jxH} z7j;;_!p5JY>~O?PlS~1Y|EL4CP^&#xq&)H$|-39&5Y~unyM$90#!Ocz{i1Aa*KK2^;ukTt#b;1M57Z0ZIQ#(0V=& zuGg5;N|6RS_6eEy*$LTfy&O(3wHdetBO*q{mZLmn#bIUIa*JD#5ytT|E5Qn|5qjHl z)K{0+cswOFZg%UOFKB~31g73%PWb$fKE_xlrn4Bz%H=ruE2{ z<~Aei)}?7b!RkrcuhQ^U+QVk-omepS&RAA1HFkBu@fP4|AywoW7%-!B5qyWXv1f$4 zp1a0o{GC=6b!6Sl?kZ0+oSt>6v}FFGrD4jj&IxPdipTmzWAMUdQQOWSFLnt@QbN@c z{qp3lk(k5kveIgch>>_B2GH>^Ei{7WM#_|f=LHDcWfUJ2Yn5k2(6+xrW!?D9_C^PF zE&Cd}Mlfs+f*yom1t-`qt3XMJUb^5FW_8jZriCgi&4%Q&0ph4E+r>*(xlV4vd}9vH zMq~LM&=&{di;+TiIKrO|Xw?x%N2Qi0@RC-T1g>J%96USf8hU*pgoXwrBO69X8yfE~ z2d|f6Pz#%O^X{BdR?<(_JB@a`WbS2NyWt?;93J~UR8ZxOl`%*i66;#vg!tZqRYhNIT+w(T=ou^j%?gLYfBxCWb?@=Lf~z~ z@R&5VX^q&0!1A?%sccc3W%U(#Kxr0Us56I_W>1DZxf6N&m1U3O;OEyzT@I5zm5L2W z^pf8^Ern@VJPleHqq{W7cnP1Z;9lCdf``!>R1tqkr2Lg;bf3?Z`J zNO-WOcZJ8Z^Zd!GX+5vHjyljQWzdZ0jOQcZ*%>VjYe_G);b*+1h+z3O*jhNNY?ljn z-ww}*$)sS{EGwUJ3~!pd>K#wDGj}y_hhn#T{!G7>`L2d``*%6L)I?#mVHZjGK>K&u zwtQ;^fO}DyTEo5Tht~$0Hu`rXZ!j+&vewv7ZiP~I+EjQ%5ckTT*Kh1-ZWk)smVFMoX^r;t z%h?6UT|e>y2Wf-~D_g6#DmBr_z&X}OfuMH{fv)$xRZe^A#G6rf#`>tm+>pyJEuL{k z^Qe>DZctE^XE*pHTqxw;+-WyM?~Et6!)lw)*3wQ0nw=X3+AH7KD`Z@g^l~Zi>$?i- z&K7QGp?ezRqz1qIw>~7l!)P+D_~>m_O2A9kpnNJ9*ySq-zX)3K)(QH;@OQb==lKFO zY&681pNhi_tT!h}{nrtnExZpNa65C28@w;K{gg+XHjd0AS%(o7c4eUF` zz0c^GTc15^M(}2(X1jvwebmm%<-ePU&lER(Mq7^^yBNlUh}N-P3f2kfUp6LfjE6c1 z4Hifj2<6x5ona3n6=-k5a9kv}GeYVX$vUntggFeza_f^F2xM=zKI>}*PaR6)j}P&cUGA zb^`#R0w;Jn%o259r#?IK$<2c2)d`-Csx+3bwQ9{b*io&up}~%3uD$H*BuQ5`w^=R8 zYRPM6wXIV``ZuSt)qI^)b{ExuyV<~{PLg6uOob4m(QhFG4{)rpRozg)o0uU z=IVV<&}yjBHdJ4?efRU__aCV!8mU*7XcDZXx#nbLk*@9f%6fX<)o*l2SU$;=G^p&T zw4=X&X9M|#Ao`xZv%YBQd*;)Ja(O|GYt+^kDqY;Oa`s2O{OOAy2IJY{fq2iPx-I|y z?Y><6*zf9#>ic)!{ep~0|NSdoAm#15KUdW~{q>K7*{65^cZDFQ$CSUm`vtGZchZaQplSneP1W9brQ6I7PMQ6jj_j%tn)PHY?`S z!DBft%CGn3WL9=SEWYo=Ld)%IVxd{mq>HAT!EmhN6A9swNj*iuM~ai*IXoqUGAn$P zS>-gWqN8tO6^IS2;^H;2if+v+y1@4xKh8HRb(jX}E)$ zQ34IBtEd=KiL^bC9CW@`DyJru6-L-jD#7baDl4R7T9b+?Mw1WK z^p1YJNioAWc{-YRkpP+g$0@3Ax1?N1))uJr)o3-tDL}RKx|sA1WFS_t^)2ai=9JTO zifRf}r1qIpRyf7B<`i3u=EYz(E9b@CU^4hv((lm$rG&myiWkOEx{@j>N~JCEiCu!{ zBNcCHlP38#(i%67@d=VjFhT>woc)}hR7_|f6=gD~tZ<5J%_*+wlmUtxcf`NrMqmImHLi-g1=U@iB%pf7;{cW27dZciv$JC6x@B zQ%=k&wxLez#M8FqOe%>~d}~tiRGuBtJKrUt*uE2rB@WPw60O4+Bh^+M9)o^W!P&wo z)Y=hJ&g20(CGBG(PBE7JYBQ&tI!>{=DftPxpt8y-p*5$3bgRPOt}KRw>1`K`0{Q}K zv_UU5WK}y5QpDnnvTQX)3NRGWNQ=)<%SMuX?sI^0av@oPyi+E66K+0Nlre6k|Xx#2zc9mywH9lu*H?(n%!GpVaCWf{0G#IOX(d5AaPXNo zb^k1=Vl)|IwlkR{sm$&x>YVL@Q~V|Lq{P%vPf?1i6CdpZTSN(w2FucSA&Zz8l*A+m zzv^_%RoM=r6lhY9WtNZE7^QXZEa>9H7{QL$Z<_PzgiS7>5nqP5=tj|a4ceG)x)Asg zA8$kmp_Y!F2zsF+2c#KiAg4ViCluXON&%TelvP4$-8T!Sz*jk|5KRUC>_PDfU&WY; zDVbUJVV0n+Ou7w)DUbpVWEP1m(DyNE1Qc3rQ=UQ=kH7 zwc$3C$y@cKXq`2mXEynn^*6vzn3!ZX`Z)RFBmRg<@f$=%L7!Zap0!B7cyUushM(>R zm<8S4o;p$q{clXcL}n@WsOaiYg{F-i7pQ0&337GDr0E;&qN4@B9&@15NfP85xW)K9 z=(EZ#swKCmP~3bhi#uK14Ccd6#ccGSC1pivoNzKMPfsuU0z{yo!g4m*__%C=(=n`) z#FnCM5E5K^c<`7ArGX!h!HiuMk21NOC<|(6R?)&kZhl)O7u}Lvj4#sXU9$>)a?|nP zzW5bC<NR^;N^r9~EeYPX#XN8@rzEHfM7l{OiE!)&P>c2UuV3ox#&7lNfai;jLF zKF4IP&QAz=X_Cnbatp>lw)TUI!Ye#hbJgXoV;4<_M5dP&dhvVG%O8kdMujeZ6J?RE z1sW3iM|DG9le02$&STj{b_DSPdT>qMlTXeortYSto8C1xw-~n=I*iK%3vXjmm8s?A zl@`}fX)!UYMXS^j+C?tfx?(o3?(QjQnILM4!NdF$%?k~?%8V{REK=4k3%P-+w@^3a zQdcc2ML{Qe*^YnrggCcvo9x zQ|k)`K*>1{0-d2cb4?c>w@s}tpmUzILN=_G!fvuL7}L}MC}!ngI{eg*Sj=`}F)XpX zN{u!Qv6#-wyV00>?3tE&s5%fa!bRSXlKTCXhPu2x5UX-uXcf59vwptI?ZGFeje}Xwr|2wwYj72u8gwOA44J4HCl)OH7IIYxB`v zSv|~)*_X1s&s}xGBwFUwULVg;lMc74+;!f%Gorxs3eh7s3#$<7vN&Hn(M-9 zt|>o4+RrqzLNohz?~Z1(hqAEJ7N5S{)BnE~_k$@B%;_oR68dzdsppI+xv66pC8?u@ z8w8NFG$b|V2E1@%JxxIFEp{!`$z<|2$`Oc|TV7VVWZxFuiZ-VHI$o*U7Sr;6jF2zJ zgAVva%S#w8q6Ml8y~roM2xAmzDz#|p<@L#74wMsydUMnCq9kJ%l?RMEU=@~BypXx? zf0b11tMZ`eio4-`F|R(ClcYz}GQ{cN^kv-F%>bNJ85fft9qH_=9T!yX!-Blw*5u?Z z1(qmmSTx0uEFIlKmL}aoWqGIrZfSJxH~!kvpY%SJ)AzUKhXKBmJ3ahc`{PRT$GxlO zweMVUUts{##d_6CuyhHrUax8>j#;$5NUQjK$SL8aPjqBGX-%{T9w{osJOR+fQjkXL zIj=IeY~_~LKDkmAHxIMXq@2xYk=bK8F3PWzHFdET{sKB&FxseN51Nn;%gu<3UsdAo_^74Q=4vPmmTcV+A~+`0-qT_UYY@TEEoHSPERcH0?IC6 z>MhPn1fTW~Da?$l_bYo-w`4}Nx!Kk^m2?6x)fE1a_N;U+x}3vobj*p-4ti(~qe>9JA%+ zIjlMJDrSz^$}z32bfqmu^I|ZYmGk0mFd2L-={M?tVnY9`FTuHGaul?Ng=duaV0q_% zgV)M(+Zp(!^FKJSAve&So?RN!BXxDQbo-KvYfUb$_%OH|jlUJ;BW)_`aD%+-e~Khs zNUqy751?Xg-R*^!ieFWP$0Kb`Gj+3zs#gJcD(~v-d)WoQ_@*&NaG7AX5{z$6FrKQq zBl--wBo})DOAKmh!C*pD>BR^OX;+aE;N8Mi*9s+J z)fXm{%ZbQEHbyQc*LJs)OKeRpv8X0D6|xG&F12u!)1tkW%=zWJgQe=fA zkRBuRP`fqAAguH$MWWlv)28T5FQ=xLCU((RXCJjEyRhC1+ZUwC;;t;`L@m>*x+`v~ z>N8Ckb-*we&{Y?eu6uPgDbjUaR|^u38VbAG)!tJs3JH+n7rYsb%i+uE`NcN&?1Ia= z(RPAq-Lngj3@@j*GyLzIJnE`RCwpDh1$bXZvvjC)%GGsUE5PW6RU=ABoyMu76mg+9 z1sX%j^l}!RFFu4!FI(xQb=xkeVl)|IPBfV#zs&9{YS`_9UM|4ZMZ>cc9c5iaNd_-` z^(C!Hi?EoC)~5FV#+#Tr<<7b;j9pAq^~LA#WjnvLZrlZ3d>AA2@k&lZM|mavUXhD) zHbT9~C@3Em3j$(r(3BYksIBO{#miWw@5HZq?Q`VPf!AdNvB+F^+0HAiJ9fd47sITg z{YLmI2E`|QJ!7i8WP;gek+j7aI{99D+74QGzaZ@d$*a;N7)805wU+oaG$P>Z83B+X z8%A}ZOgDuuU~~Agon~4$?Sd_a0BQ^1pS2B0ioxZf&G;m~9ZrR<=GO8uZs8HN|EQ>o`+zjT! zPxbD4`~}jO<78NNU|M$pOOB+3W%b25Sv$`;p&S@CPYO=hP}g;}a34UGyg|;$|2lvIV_;HWs-@1%x)CIKLf%QwZS_V^>f2rC6 ztNs34)M_f`#3V#T42+~(_%wjQI4GpWNwZhD3u@6=Q20RO%(KL@Fb(WMm(5<1d_?zdEw)2c@moi!Ksh@W;9F5B< z5zcIc*W6_I4Kt~7*hTplE+C>Y`e`SLiOa@b#-tXmn|gk&$t9ZYJQH8HFIKr?T8$}`d8|Iue{M&!Vmx{*SGKct5Z?Iyhc`~@MGp-g zhBBtojWR*8oqAv;1~j^!)yC)0Zec+q$uJ^M8FbL9ry4>V8FFZ|eKnhW73CO=aB3hF zvvM#Uemeb9HgJ7k%0@e>&2kf41XUA^6~=1CJIFv3(YreFQnc*8CL5(ha53}C$y2Am z8~2)ArP)O2GrcRX6$51?*4q-Y(mK!9A@~ z!rW-pacb)Z$mn5_H6BDRPysWjK_Z-CiRlu4 zZ9cjytA}|p`%;$oxm|Csodn>PRT>xNvaQBO40&|JK5J`i-ASy-6VEQX{*3TO=CvnX zFivP-9B1>a({{$$w?lX|n?00;mA3fw<(~fkwYVQliEB>JF1;t0C_=1fvK5|w5thP} z4VG`EQ6zAORF8?CO2Da;CMdk1^L$e$)!?R`Cr)MSs_o3OZ!>U3OJ#o@uM~TWX?Z_J z0w~6V4%kJ@3s~7^pbkLDbV}Am8l)HJH6Ods%9AJ;&BQ?%r55&`QE2K9&IEIEf@#`0 z>#{fA4uY|-N|vH4?uPfpy!u>DlCVviBBz7XSAbtPd2mh@U?d11QXjA~LSv${;29&= zy6v`CfSt58w_pe)t+Tbqj04CG!5x%pJC6oaV|MCEgNr33P4icwaVD`oQ7Mjowwe5ERG9%iFS zIh)atQ{(Xg?N#+ul+8{3%Jw$(5F?Y-ak+wVUxYt}ro)_GjVbw2kE-|*!pcfT>L z0UoGct#TIT4f-V=sJcHTBjd50=2X1>kg1Kuo<=urk zE76qM6qV!R8b@&M(&vsPENwAbvsz7EoFKUBDS&_@+yqe$m+L=Tqw!x_V-&I-$EsLq zO?z#A9+v@B%l=EAsIcIpIlVi?Z+Rj{=il?QXyFzPYGh{08!F*Ud^YR@n3&%lVf}H*xzpTjEbi_s8E79?MLo zx-KfmzCx+XzY_vRi5WfW&e;CR=&H*wMKNY~?PuX_rZ3XxkGDh1Fr$`EE}!^<(_ucB z%M&u%*PXGzh+V(ch9%u>l;UA#T{uS;AH?DrM^E;~hZ;h=h$yO(SY&RVXB#i?;G#s1 zxKw0(2Jrm)ckaSQf7BAw7qO(XM%}f{Ck|k9FSo~`x2CYw0RW%-oPAqD-;^4`YShArRYCFR8I_-heHsGolnt|8MN&~_mA z_XS%BR&-$A&!AmN!X1kww)yH(^Y_`laiRur);)4u_YXP$DxekL-K&OOTUV}?qVJ_6l?9%7l? zuXR!U==4wUu?<-TB(J>(Uv1iZ@X6Gl8&-q?0v2aKqG#BB@xbEYAsM)t)BLiW~% ztj9~di7>L$J6)#HhC4Y=B9DG9A*f+YPYeYRk1*HZH9;?k7AuHxc)-_v;=K=Od;A86 zsNt`P8+#biytB9r;#E*-hDo4%DnkLqTbC- zS=}jPYb?LpL64_D3-6~aP+{{4**2}=h^1F87h|Eh${`y@!cS|*2b(F9_;a4Rz*FyY z2-=>2ivUwc!}`pKDS9oSMw#0tL!h1TGryL8ZwDJ{$q)=%+PC+SIX`` zl4(9Ymq1guY}vT!kCsmCQp&W|72ELhdI{Y+6P=rVRYD<$;;c^vZkCM(1@Cu}Jst4| ztR39%!cn;L*EVuXCmP56c+ZFMn)iID$_rOWTEz{LMpF{UiMs37%Ig@=(U1Y5w3x&Y zEY3AihaC_K?>)O}gnhTJBU@Fs%Z@y{-sV+ZB8`viteAv(J7saxi+HQ`bRWnpyHZcf zYOh@H{Cn`l&Fe$fz1(n0JBhG-;cubn-2_p6PgJ1-A*%8V1TpEW`ejxfiK!mwhA^n% zN6YQpd6&}voqXly{x$g$5CJ(F_uz+I#if%dD2Fi6r=i89=_Qrg$U(BkJD}~MLAo5O zCA`hNNM*p#Z!^-|z}}#r9C#UV5p{;G7l|8iAm7WrY9hP-7sOr7=j$*shZ+Hd zxEZ$Jjop~1+dh{yY7R)0zkbipq5r5&JC*l-e8b4hR9iUXm`h^*5=K67@XLn6aJBek z_`@LG7@Xwn`>7hAcAs6Rkn%lS0+CG=6sN~|nJsYbvH5G#UOf{Q`X$QSP5$qc?0V=M zW+jAEoO9s0hSG$?E>*;HFle~nP zqy)-pR}ZeRlt**G%~tbZ{T{idC+q>iZyh5clQvT480UXluibH#XQm-SRNTbt!7~ zABTsDS`s53Y6Nx~Xd}bTfSI?*4q7v9VWjQU9UFR50hqlvqdaBMp^~&DYRzf7MY$>o z3o6BfQ!^&y{F>xjy*WjCVlFPum&#USV*iZN=%^Cn_1^Q47)xsyvH6?DfPqFN$?54I zF6`u!tW8SmQrsFFjJqymVGx!I5Gldq5yA3e-wGDV!n0QrFHw(KtGx(P|5*`Ga{hZo zFe9x2rwe*!$)daX73TG4oV2q^y^b47jC^elhl72eOeH3sTs(PLoEyG;+kiVGB~o!% z5(4EgWoL%t!-Aw#IY9+voa6qDOVFQ;_z(JAhsKqbn#PzXLl*P0DWF{f3siP)XY8M@ zrAT;|Z? zzliDWn!WkHNoEF$#>fN?K#!I9q6vB{7@qY7iLV(*gQo9csElJuOeqO+b+vI*nh#c* z2CooO7d%UE!={^bexDZ5EQ%jVw8=VRWjQd( zjm%~^w)vi4ddfp}pv?~?O!Ecu=_#w25c6vi*GzVaC6#Lh+c^u)wv6E;^VyBk1D*pu z`zU+%XtEcms=x(MPUKj5bu9*CEPt`E{k=21Br#H7H-4Eh54@(FLR%C47?vOlY$@~!-B_?Y3Q3WmC0zN57P@NvxK-RQ3w_JKF zx#`y+QZe0=Oyt~e!Q#?EgA3STJ{mK0emaE1f|ZE#>L^(~9E4X!9ezgtAivBy=|$#g zmi65yYj`i5NppOSk{|P8+E~X&_4%SP2DAtX{lN>XbJ?+|<%$Xm*%7S85eeFdRvh)z z8dsj?m^by+aTY_e|3wjOy_V6cjhy5H?-;>Ledm>uh&HuQ;)zK;dzgyd-8@%jdCZv# z7o*;H2jE4H#x}8Hy<_FrmzOQXx+8dsK<-TM(RQ^4fpp^HE+wAIEV*;X(zm<4E*ZRg z@*W$R{R`Fv20a|$sJ?xPn|4Ot35<>{EP$9zPEh4xM-8oDmXz46A2pTS7eq&Qv#3cW zT6R4`5EU@g)PalCN5=C@pwRcP< zk^2A`hsrKyfDl0_*Fy&%G$6ANNbl>;%qxihmbD_@SPn7Qk3Nd-)mBlEB0=!3^L6{z zE&-JHcbCw*EMoOzLS{ZK&|W%#6oqkxm;wxeAd5&NW=J+V>sd4+D?5{Foh9+-&D6SW zFM46P9xcu?x$b>R$Mg6XtjxSm>3i?6;@JESD}e+*^raEmP)f;ImiVRpa2Z=eDcs2N zWyNZ{5fBP&!kwVH_T*1J4{5n4RqTRngVZ%Om+w5#r%6I${!yI;&ZVJ12i)gu9U0Eb%kJ-4?cf_y zcGR3YH*qT>p88Q*>#?P2y53`wC|vxI@dz}xvtInVHSb!lfK+LfOX=STDx;GTb+Tfk zlik#F;O@~u6Qj1c@81}bgLn5njX;u#O!z%s*?gP_dT6m88^)Tkke`||x=nd=6iCI| zo-%5#k*)R*8M+q^=77YGl3GrYSfGTa(&ms>CS{eCwUTViQ#$hJ?CP(f5lN44D~ZK2Y- z=%+sWpu-VMsWkKpOf~Tu4{8n!Q*QN8zCwJOPlKs|*{uQ>mxIR| z^5OChk8-8lHdKMVzqmp#x+C|?vZs@##;0q;yL`rT$B`XjJl)x1%^|o>tq#+4Ln&Lx zwisbhjk@d-t9*%>JZs`L8-|*a-ZJG9|HW;7DMf)-TC-H`l9;v@FkDIh?C_6%_;Ysx z?cojO&JSDN(tLSgO5Aha;^Q?LB!wtS4J|#6Xh%O&xoHLbr{Frhsh;C^xWYR53ss(ty1mhV)L`=(KNcr=*G>g>?L|`UWUr3NI)x4?my*;JgS)J-6q;jRdP+@;5bexnGOnB z{zPC}PDylmT2$FRl9V+|o}mfXaq%NhsGS?ehx}zncEq&QMh=P$I*MXkf}~0b?}*>v z@b?CpuY{_9zi18GEtlS_>Q!GWjPo zb38{3yb>TvwLQc0x_SjtETo42uOR^RHE=$kf9ESdepN1?RaNbbMgM6PtGXo3x6~!c zNecyRA+c@wj@=s5B^DJi-W;|1UAQOu(I4UMXmPO7ltO9l`2BTKDO2)3#23ACBEBNu zF;n0;BR6s222Muw_(fQ1UU{rA@QEAo)CqurDhR7hHW8S9?dD@HJ?;A`qRdrON+O2~ zEAYKBpwyR57!d3TK@uO#lK|1t-Bb?sC5>Q!)9 z_Yq#^-kX`yt&!E>gR-2IkM&QYcxTK{hd(;GPi4an2{L+E^*1#+NHUm6nuDI|EsFI0 z+utAAY7k0H8Sr0w26?hl8Jy#qFM7UpwrcV_0dTeEJ_(pa_9=kuLz@&+QBQHh9<8tj zLOy`NFH1?YvSZ842#qxIMpFwr=Uq$l)j*UIT!o-tSqgiDlC{raPnE5t6zk8EgE6UD zP`RiU23ZlW?jO)dF2L#~9u!Ox!Ac??WaW0~q;fznJ%;JB6F4s~R9}?q6ki>)9Yaw04=Z#c`yN8pSC_2i4KkH@-r1 z1<_QC1D6QR0kQJd%I=uN!Lr;r%Lk~x+Xp(M5PZ@A z*Hn4Z4AQxPBB0w4s0*GFKF{m#A8DwlJAxZJGjWe zPi<^44l6`w#{`#j4n;DmV5X^wHsrhRvcz%jWa?9$nl5JgL|-n4O{v z%18ZGKaMFrUj^AhTSnD0U7X6&0lnMy8UT$@{5&{whLF(cPslHy*f(0-Fany8KFl&5ApxxZvN=tPm@q;485+ zVtx72^-StV75A~3xJFAaiC?n`b&2>+En~envdxPg*l}@4N-mq8R9BRwq{WD=1~h8! zG044Ry;UZ~uC!7&?_UZ~72-+5706`WLe#r<&5yxyB^joE3xCS(6!p z_0$`4?p5OMM(!wVMdsBgaQQB4Eq_v*;zYgf;2kFM)Q_akZijluv~BFwxAUuBdBnZL zPQt%ohnWKZ+h5q}ud8Ke!}>-b5xKHEeRTE zi{;okrtv1i9|~4$1u43*lA=XhS!MohaU`TM+6<>vDew&lToPN0-v~ruQ(XMabHt=e zs}nB}Sw3^ihYbP>8U(_6@TIB2ZfcU}CWB>84fd)t^8>hU8te|l zzskf&VqXkVfPBY2a=xxzJp#h?D0#jk+m#%!xaFn}89Xh$k~?awKwK-dKW2&Y3hp3_ z?S*(;h7Y?XT~bhn%`3#`=_@0Y56)8TghSUozZbz@Zrr1@?o~3b28jPieKT8qNrc~+ z@g!Jn(D)+sUI3ffF!`qF(Sc#3ASBy`5r?>W1Yc~z&DMD-D_+QVfK+S# zVpw1h>Q35o8j`R!2h{I-453O{;zxK%W|FOKRns6!$9vEKHM6iZ)edNd|;{{pQNJj5u;;eJGbDEhc$O%ex)pSStv(72|4;jgzI8GfM2eneY)h*+6*6E$d`h9sx z6b8$3<5Eet>W5N_ym@Bu|Jov*oqyS)(L^Fv)jt^xbnvdlFkR4{RSI!QT44s11P5D+ z@{&B|@`q{hPzl;ZGD8tgHtah7edQWuZQ9FLzRzAk4!Z9?= zc((riPE1UdeIg9CQ+S%SdcA(!$L{Q53pn`ocgsl)xMShnRvP2st=+~K)gb+X56nlM zo#o;DFyb2GEb*b-kCc4EoDPEq72mXv&9^9L8P?Jv5bJ8{TpEliv6Lj4FRY9qE21KG zEx8@~g1cAEU&?7ofHGi*4ZmC*mJBHbu82)3GiC3V zSw(9}>twX}mxYk3@ibt;`cjb5brmOc1&W9Q9JW7Zwm@8Q4)VF;aG5_!au=DYd?P@s z-XjMmiw*y+Py81BpZY|BvxsENJr%j#{48Dyhw#_TdRB!yaz!MFMEr0F76b>p>Tk+O zBGh>t-@J|S2jTGnM7whm5|b_G_@qlSivJWtwZ$~zndeQ#{v=loWn0W}GWoey*?s5e z#y3A;tYQ0_ctQQ!oUXGsP4VX#%rQr1g0=Dl^7`Jv#?xYv^ETH%JqlCq6bhoT_Z~&_ znP23StMqPhq;fPr3i)keeV8+Fy$*4EzWmI`iO3k|z6bA=W@u`1 zNmW@at$zAMzRVBD=p}hVq#Rt<9pn5?G^hVVH1t@`A`(}O)Sp4{gg8R$?Q{Jh*xweA zqPk{}7YHS#hObJ+SySYU>d@2#)heSwX`Jg>RCU)}XSOky-PLSpzwMmah3n4hVReuP zKJh5Eiw|s)xjn~7BbslJ4PD0=TFmzcWFLEzV_6cH1dq6hag81rA_kK00*j9n)+`5@LcMD=>s+EwXW{yn9K>Ut&Uid}zA`>B z^*o;?hKUs@s_anr9hO$@%1X%)qj31!(Nn5(+z%^!IOJa$Ri6o7x63QzG`is-`U_R5;WEPoRKN^Lm45eoAAdm7Z ztsY4Q75KO?p|q%HOJjP`A46fFd;npwpFqP`ewK&?LuYA+O7ZDV1Q{Z=ldLaQZUj@X z;H=jmMqn_v1LAoU-V+aea%aD(>Y{T5Fan)|J}VPi?{gZd;bkNBSE;%g5-Y-JxG`WD zW{K97VIHyn7`N@mc}?0!U|<}363V`}4+#Izmq%WnGuOG{RQDCUOVSRrKW(LBp2AG& zJ2;$xD%1T4ip5-6!VKT3j&#&w_-eM?M^1