diff --git a/integration-tests/build-tests.sh b/integration-tests/build-tests.sh index a17cca6a..73fc8a2a 100644 --- a/integration-tests/build-tests.sh +++ b/integration-tests/build-tests.sh @@ -378,6 +378,12 @@ fi if [[ "$BENCHMARK_IS_PROTOTYPE" ]]; then echo "export BENCHMARK_IS_PROTOTYPE=$BENCHMARK_IS_PROTOTYPE" >> lexical-graph-examples/.env.testing fi +if [[ "$BENCHMARK_DOC_STORE" ]]; then + echo "export BENCHMARK_DOC_STORE=$BENCHMARK_DOC_STORE" >> lexical-graph-examples/.env.testing +fi +if [[ "$BENCHMARK_S3_JSONL" ]]; then + echo "export BENCHMARK_S3_JSONL=$BENCHMARK_S3_JSONL" >> lexical-graph-examples/.env.testing +fi zip -r graphrag-toolkit.zip graphrag-toolkit # zip under directory diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_extract.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_extract.py index 510dd054..d618d9a5 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_extract.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_extract.py @@ -14,7 +14,7 @@ from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory from graphrag_toolkit.lexical_graph.storage import VectorStoreFactory from graphrag_toolkit.lexical_graph.storage.graph import NonRedactedGraphQueryLogFormatting -from graphrag_toolkit.lexical_graph.indexing.load import FileBasedDocs +from graphrag_toolkit.lexical_graph.indexing.load import FileBasedDocs, S3BasedDocs from graphrag_toolkit.lexical_graph.indexing.extract import BatchConfig from llama_index.core import SimpleDirectoryReader @@ -69,9 +69,33 @@ def run_benchmark_extract(handler: IntegrationTestHandler, ) indexing_config = IndexingConfig(batch_config=batch_config) - extracted_docs = FileBasedDocs( - docs_directory=os.path.join(data_dir, dataset_name, 'extracted'), - collection_id=dataset_name + doc_store = os.environ.get('BENCHMARK_DOC_STORE', 'file').lower() + if doc_store == 's3': + missing = [ + var for var in ('AWS_REGION_NAME', 'S3_RESULTS_BUCKET', 'S3_RESULTS_PREFIX') + if not os.environ.get(var) + ] + if missing: + raise ValueError( + f"BENCHMARK_DOC_STORE=s3 requires {', '.join(missing)} to be set" + ) + extracted_docs = S3BasedDocs( + region=os.environ['AWS_REGION_NAME'], + bucket_name=os.environ['S3_RESULTS_BUCKET'], + key_prefix=f'{os.environ["S3_RESULTS_PREFIX"]}/doc-store/{dataset_name}', + collection_id=None, + for_jsonl=os.environ.get('BENCHMARK_S3_JSONL', 'false').lower() == 'true' + ) + else: + extracted_docs = FileBasedDocs( + docs_directory=os.path.join(data_dir, dataset_name, 'extracted'), + collection_id=dataset_name + ) + + logger.info( + f'Doc store: {type(extracted_docs).__name__} ' + f'(for_jsonl={getattr(extracted_docs, "for_jsonl", "n/a")}) ' + f'collection_id={extracted_docs.collection_id}' ) with (