diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..300e53ba --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +source = . +include = */*.py + +[report] +show_missing = True diff --git a/.env_sample b/.env_sample index e370568f..cec3ed13 100644 --- a/.env_sample +++ b/.env_sample @@ -6,13 +6,33 @@ DJANGO_ACCOUNT_ALLOW_REGISTRATION=False DJANGO_AWS_ACCESS_KEY_ID="" DJANGO_AWS_SECRET_ACCESS_KEY="" DJANGO_AWS_STORAGE_BUCKET_NAME="" -GITHUB_ACCESS_TOKEN="" -GITHUB_BRANCH_FOR_WEBAPP="" IPYTHONDIR="" REDIS_URL="" -SINEQUA_CONFIGS_GITHUB_REPO="" -SINEQUA_CONFIGS_REPO_DEV_BRANCH="" -SINEQUA_CONFIGS_REPO_MASTER_BRANCH="" -SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH="" SLACK_WEBHOOK_URL="" USE_DOCKER=no + +# SDE curation pipeline (all optional — defaults hold when unset) +AWS_REGION="us-east-1" +SDE_S3_BUCKET="" +CRAWLER_INSTANCE_ID="" +CRAWLER_INBOX_PATH="/opt/sde-crawler/jobs/incoming" +SCRAPE_POLL_ENABLED=False +SCRAPE_STALL_TIMEOUT_HOURS=24 +INFERENCE_ENABLED=False +# pipeline-scoped credentials for local dev ONLY; leave blank in AWS (instance role takes over) +SDE_AWS_ACCESS_KEY_ID="" +SDE_AWS_SECRET_ACCESS_KEY="" +# only with temporary creds: aws configure export-credentials --profile sde-dev --format env +SDE_AWS_SESSION_TOKEN="" + +# P7 indexing hand-off — dev-only for now (sde-web-copy); blank = dispatch disabled +SDE_INDEX_BUCKET="" +INDEXING_ECS_CLUSTER="" +INDEXING_TASK_FAMILY="" +INDEXING_CONTAINER_NAME="WEB_COSMOSContainer" +INDEXING_DISPATCH_ROLE_ARN="" +INDEXING_SUBNETS="" +INDEXING_SECURITY_GROUPS="" +INDEXING_ASSIGN_PUBLIC_IP=True +INDEX_POLL_ENABLED=False +INDEX_STALL_TIMEOUT_HOURS=6 diff --git a/.envs/.local/.django b/.envs/.local/.django index 0978166d..43069e3a 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -22,28 +22,37 @@ DJANGO_AWS_ACCESS_KEY_ID='' DJANGO_AWS_SECRET_ACCESS_KEY='' DJANGO_AWS_STORAGE_BUCKET_NAME='' -# GitHub (please create a new file called .env and put these in there) -# ------------------------------------------------------------------------------ -GITHUB_ACCESS_TOKEN= -SINEQUA_CONFIGS_GITHUB_REPO='NASA-IMPACT/sde-backend' -SINEQUA_CONFIGS_REPO_MASTER_BRANCH='master' -SINEQUA_CONFIGS_REPO_DEV_BRANCH='dev' -SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH='dummy_branch' - # Slack Webhook # ------------------------------------------------------------------------------ SLACK_WEBHOOK_URL='' -#Server Credentials -#-------------------------------------------------------------------------------- -LRM_DEV_USER='' -LRM_DEV_PASSWORD='' -XLI_USER='' -XLI_PASSWORD='' -LRM_QA_USER='' -LRM_QA_PASSWORD='' - -#Server Tokens -#-------------------------------------------------------------------------------- -LRM_DEV_TOKEN='' -XLI_TOKEN='' +# SDE curation pipeline +# All optional — defaults hold when unset. SDE_AWS_* are pipeline-scoped credentials +# for local dev ONLY (distinct from DJANGO_AWS_* static-assets credentials above); +# leave blank in AWS, where the instance role takes over. +# ------------------------------------------------------------------------------ +AWS_REGION='us-east-1' +SDE_S3_BUCKET='' +CRAWLER_INSTANCE_ID='' +CRAWLER_INBOX_PATH='/opt/sde-crawler/jobs/incoming' +SCRAPE_POLL_ENABLED=False +SCRAPE_STALL_TIMEOUT_HOURS=24 +INFERENCE_ENABLED=False +# Real keys and the dev resource ids go in the gitignored .envs/.local/.sde-aws (see +# .sde-aws.sample); local.yml loads it after this file, so its values win. This file is tracked. +SDE_AWS_ACCESS_KEY_ID='' +SDE_AWS_SECRET_ACCESS_KEY='' +SDE_AWS_SESSION_TOKEN='' + +# P7 indexing hand-off — dev-only for now (sde-web-subset); blank = dispatch disabled +SDE_INDEX_BUCKET='' +INDEXING_ECS_CLUSTER='' +INDEXING_TASK_FAMILY='' +INDEXING_CONTAINER_NAME='WEB_COSMOSContainer' +# Blank on a laptop: the dispatch role only trusts the instance role, so a local SSO +# session calls ecs:RunTask with SDE_AWS_* directly. Set in AWS. +INDEXING_DISPATCH_ROLE_ARN='' +INDEXING_SUBNETS='' +INDEXING_SECURITY_GROUPS='' +INDEX_POLL_ENABLED=False +INDEX_STALL_TIMEOUT_HOURS=6 diff --git a/.envs/.local/.sde-aws.sample b/.envs/.local/.sde-aws.sample new file mode 100644 index 00000000..d6394758 --- /dev/null +++ b/.envs/.local/.sde-aws.sample @@ -0,0 +1,22 @@ +# Copy to .envs/.local/.sde-aws (gitignored). local.yml loads it after .django, so +# anything set here overrides the tracked blanks/defaults. + +# Temporary SSO credentials. Refresh with: +# aws sso login --profile sde-dev && aws configure export-credentials --profile sde-dev --format env +# then recreate the containers (docker compose -f local.yml up -d --force-recreate django celeryworker celerybeat). +SDE_AWS_ACCESS_KEY_ID= +SDE_AWS_SECRET_ACCESS_KEY= +SDE_AWS_SESSION_TOKEN= + +# Dev pipeline resources — ask a teammate for the current ids; they are not committed. +SDE_S3_BUCKET= +CRAWLER_INSTANCE_ID= +SDE_INDEX_BUCKET= +INDEXING_ECS_CLUSTER= +INDEXING_TASK_FAMILY= +INDEXING_SUBNETS= +INDEXING_SECURITY_GROUPS= + +# Optional: run the S3 pollers on this machine (tracked default is off). +# SCRAPE_POLL_ENABLED=True +# INDEX_POLL_ENABLED=True diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8b8b6d5a..94a0dfc8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,7 +11,7 @@ assignees: '' A clear and concise description of what the bug is. ## Steps To Reproduce -Steps to reproduce the behavior on https://sde-indexing-helper.nasa-impact.net/: +Steps to reproduce the behavior on the COSMOS web application: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' diff --git a/.github/workflows/run_full_test_suite.yml b/.github/workflows/run_full_test_suite.yml index 10c61336..a18950fe 100644 --- a/.github/workflows/run_full_test_suite.yml +++ b/.github/workflows/run_full_test_suite.yml @@ -22,23 +22,23 @@ jobs: - name: Check out merged code uses: actions/checkout@v2 - - name: Set up Docker Compose - run: | - sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose + # Compose v2 ships with the runner's Docker. local.yml uses the long-form + # optional env_file (Compose >= 2.24), which the EOL v1 binary rejects. + - name: Show Docker Compose version + run: docker compose version - name: Build the Docker environment - run: docker-compose -f local.yml build + run: docker compose -f local.yml build - name: Run test suite env: DJANGO_ENV: test - run: docker-compose -f local.yml run --rm django bash ./init.sh + run: docker compose -f local.yml run --rm django bash ./init.sh - name: Generate Coverage Report env: DJANGO_ENV: test - run: docker-compose -f local.yml run --rm django bash -c "coverage report" + run: docker compose -f local.yml run --rm django bash -c "coverage report" - name: Cleanup - run: docker-compose -f local.yml down --volumes + run: docker compose -f local.yml down --volumes diff --git a/.gitignore b/.gitignore index 24e03b4a..371c107a 100644 --- a/.gitignore +++ b/.gitignore @@ -282,6 +282,7 @@ sde_indexing_helper/media/ .env .envs/* !.envs/.local/ +.envs/.local/.sde-aws **/.ipynb_checkpoints/ **/*.xlsx diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0f4c06a..4fd8e657 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ exclude: "^docs/|/migrations/" -default_stages: [commit] +default_stages: [pre-commit] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -12,7 +12,7 @@ repos: - id: debug-statements - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 + rev: v3.21.2 hooks: - id: pyupgrade args: [--py310-plus] @@ -61,7 +61,7 @@ repos: - types-requests - repo: https://github.com/PyCQA/bandit - rev: "1.7.0" + rev: "1.8.6" hooks: - id: bandit args: ["-r", "--configfile=bandit-config.yml"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c0d7ca1..6d836986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,17 @@ For each PR made, an entry should be added to this changelog. It should contain ## Changelog ### 3.1.?? +- cosmos-rewiring review fixes + - Description: Pre-merge fixes from the branch review. Status-triggered tasks are enqueued on commit; Slack messages are posted for statuses set via queryset `.update()` and for the `PRODUCTION_INDEXING -> PROD_*` transitions; re-scrape failures no longer overwrite the live workflow status; `send_job_to_crawler` refuses to run on a host without `CRAWLER_INSTANCE_ID`; duplicate URLs in crawl output are dropped instead of aborting the ingest; the prod status mirror map is explicit; `print()` replaced by `logging` in `tasks.py`. + - Changes: + - `sde_collections/models/collection.py`: `_enqueue_on_commit` wraps the four `.delay()` calls in `handle_workflow_status_change` + - `sde_collections/tasks.py`: `_mark_scrape_failed`, `_dedupe_by_url`, `PROD_STATUS_FOR_QC_STATUS`, logging + - `sde_collections/utils/slack_utils.py`: `notify_status_change`, two new `PRODUCTION_INDEXING -> PROD_*` messages + - `sde_collections/scraping/ssm_dispatch.py`: settings guard + - `templates/sde_collections/collection_detail.html`: removed the "View on prod" buttons (properties deleted with Sinequa) + - New tests: `test_signals.py`, `test_management_commands.py`; additions to the scrape/ingest/indexing/trigger suites + - `gitleaks-config.toml` added so the pre-commit gitleaks hook runs (it referenced a missing file) + - Deployment: none beyond the branch's existing migration/env steps - 1232-process-the-full-text-dump - Description: A script was added `/scripts/sde_dump_processing/clean_text_dump.py` which cleans dumps from sinequa. The sinequa dump does not respect normal csv new line formatting, so that a dump of 1.8 million records becomes a csv of 900 million lines. This script can detect the headers and process the dump with the three possible sources TDAMM, SDE, and scripts, in order to create a final, clean csv. It has a simple CLI which allows setting the input and output, the verbosity of the logs, etc. Because the input files can be very large, the script streams them instead of holding them in memory. - Changes: diff --git a/CODE_STANDARDS.md b/CODE_STANDARDS.md index 39d473b7..9616173b 100644 --- a/CODE_STANDARDS.md +++ b/CODE_STANDARDS.md @@ -47,10 +47,9 @@ The following pre-commit hooks are configured: - black: Formats Python code to ensure consistent styling. - isort: Sorts imports alphabetically and automatically separated into sections. - flake8: Lints code to catch styling errors and potential bugs. -- mypy: Checks type annotations to catch potential bugs. +- mypy: Configured to check type annotations, but currently a no-op — the hook sets `exclude: "."`, so no files are checked. - bandit: Scans code for common security issues. -- gitleaks: Prevents secrets from being committed to the repository. -- hadolint: Lints Dockerfiles to ensure best practices and common conventions are followed. +- gitleaks: Intended to prevent secrets from being committed to the repository. Known gap: the hook passes `--config=gitleaks-config.toml`, and that file does not exist in the repository, so the hook fails instead of scanning. ## Continuous Integration (CI) When a commit is pushed to a branch that is part of a Pull Request, our Continuous Integration (CI) pipeline automatically runs specified tools to check code quality, style, security and other standards. If these checks fail, the PR cannot be merged until all issues are resolved. diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000..1a0fc4ef --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,1028 @@ +# COSMOS Rewiring — Implementation Plan & Progress Tracker + +> **Status legend:** `[ ]` not started · `[~]` in progress · `[x]` done · `[D]` deferred by decision +> +> Update the checkboxes in this file as work lands. This document is the progress tracker. +> +> Companion documents: [`WORKFLOW.md`](./WORKFLOW.md) (what the pipeline does) and +> [`sde_collections/DEPLOYMENT.md`](./sde_collections/DEPLOYMENT.md) (how it deploys). + +--- + +## Context + +COSMOS today drives collection curation through **Sinequa**: workflow-status changes generate +Sinequa scraper/indexer XML configs, push them to a GitHub configs repo, and pull scraped full +text back out of the Sinequa API. Sinequa is being retired. + +The replacement, specified in [`WORKFLOW.md`](./WORKFLOW.md), is: + +``` +COSMOS ──SSM job JSON──► crawl4ai scraper on EC2 ──► S3 ──► COSMOS DumpUrl + └──► DeltaUrl ──curation──► CuratedUrl ──► OpenSearch (test) ──QC──► OpenSearch (prod) +``` + +`sde_collections/DEPLOYMENT.md` is the companion CI/CD spec. Both, plus `WORKFLOW_DIAGRAM.png`, +are tracked on branch `cosmos-rewiring`. + +**Intended outcome:** COSMOS dispatches scrape jobs, ingests results from S3, drives curation to +`CuratedUrl`, and hands curated content to an indexing pipeline — with Sinequa removed, the +inference pipeline dormant, and a repeatable deploy path. + +### Confirmed decisions + +| Decision | Choice | +|---|---| +| Scope | Pipeline rewiring **+** the CI/CD machinery from `DEPLOYMENT.md` | +| Sinequa | **Delete outright** (unwire first, then remove the files) | +| Indexing pipeline (chunk → SageMaker → AOSS) | **Built as a `WEB_COSMOS` source inside `sde-api-scrapers`** (branch `web-indexing`) — supersedes the earlier "separate repo" call: the quantization math must stay byte-identical to what built the live index, so `uploader/` is reused, not forked. Event-triggered from COSMOS via `ecs:RunTask` through an assume-role indirection (same-account in dev — both repos deploy into `998871305517`; kept so test/prod can be cross-account later). Its phases W0–W3 + W5 are **code-complete** (316 offline tests) and **deployed to dev 2026-08-20** (merged to `develop`, PR #47); this plan owns only the COSMOS side — its Phase 7 = that repo's **W4** | +| Validation per phase | pytest with mocked AWS; manual smoke tests against the real `sde-dev` account | + +### Decisions added after plan review (2026-08-11) — confirm with team + +| Decision | Choice | +|---|---| +| **Reindexing (re-scrape) flow** | `create_configs_on_status_change` has a second, `reindexing_status`-keyed block (`REINDEXING_FINISHED_ON_DEV → fetch_full_text`, `REINDEXING_CURATED → promote_to_curated`) that the original plan didn't account for — P6 deletes `fetch_full_text`, which would sever re-scrapes. **Rewire it onto the same dispatch/poll path**: dispatch on `REINDEXING_NEEDED_ON_DEV` (P3), poll it (P4), ingest flips to `REINDEXING_FINISHED_ON_DEV`, and the migrate task's existing `→ REINDEXING_READY_FOR_CURATION` transition takes over. Keep `REINDEXING_CURATED → promote_to_curated`. | +| **Dispatch record & result freshness** | S3 objects persist between runs, so on any re-dispatch the poller would instantly "complete" against the **previous run's** output, and the stall timeout has nothing to measure from. Add a small **`ScrapeDispatch`** model (P3): `collection` FK, `dispatched_at`, `ssm_command_id`. `results_ready()` requires the S3 summary's `LastModified > dispatched_at`; the stall timeout is measured from `dispatched_at`. | +| **Web indexing pipeline: `WEB_COSMOS` in `sde-api-scrapers`, event-triggered** *(updated 2026-08-13 from the built `web-indexing` branch — supersedes the earlier separate-repo / container-env / `describe_tasks` sketch)* | The pipeline is a **`WEB_COSMOS` source inside `sde-api-scrapers`** (ECS Fargate: chunk → SageMaker vectorize → AOSS bulk upsert, `uploader/` reused with parameterized `chunk_field`/filtered scans; `schedule=None` in every env — fired only by COSMOS, per collection, the moment curation completes via the P5 triggers: `CURATED` → test, `QC_PERFECT`/`QC_MINOR` → prod). The hand-off is **S3 both ways** through a dedicated bucket `sde-cosmos-indexing-{env}`: COSMOS exports `CuratedUrl`s as `documents.jsonl` + `manifest.json` (manifest written **last** = export complete) under `curated_collections/{config_folder}/{run_id}/`, then assumes `CosmosIndexingDispatchRole-{env}` and calls `ecs:RunTask` with a **command override** (`python3 api_scraper.py --source WEB_COSMOS --collection --target test\|prod --run-id ` — the executable must lead: an ECS override replaces the task definition's command wholesale and the indexer image has no ENTRYPOINT). Completion is observed by **polling S3 `index_runs/{config_folder}/{run_id}/status.json`** (written by the indexer last and unconditionally, incl. on failure) — **not** `ecs.describe_tasks`; an **`IndexDispatch`** record (`collection`, `run_id`, `target`, `task_arn`, `dispatched_at`) provides the stall timeout, and the COSMOS-minted `run_id` namespaces every artifact, so no `LastModified` freshness rule is needed. No callback endpoint into COSMOS. | + +### Hard constraints + +- **Do not modify `DumpUrl`, `DeltaUrl`, or `CuratedUrl`** (`sde_collections/models/delta_url.py`), + nor `migrate_dump_to_delta()` / `promote_to_curated()` in `collection.py`. The rewiring changes + what *feeds* those models, never the models themselves. +- Every migration must stay additive and backward-safe so an image rollback is a complete rollback. + +--- + +## Key reuse points (found during exploration — prefer these over new code) + +| Seam | Location | Why it matters | +|---|---|---| +| `create_configs_on_status_change` | `sde_collections/models/collection.py:871` | The single `post_save` dispatcher for all status-triggered side effects. Four of its five branches are Sinequa. **This is the main rewiring surface.** | +| `fetch_full_text` | `sde_collections/tasks.py:159` | The only writer of `DumpUrl`. Its contract is just `{url, title, full_text}` → `DumpUrl(url=, scraped_title=, scraped_text=)`. Swap the source from Sinequa to S3 and everything downstream is unchanged. | +| `migrate_dump_to_delta_and_handle_status_transistions` | `sde_collections/tasks.py:200` | Already does delta migration + status transition. Its `pre_workflow_statuses` list is where new statuses slot in. | +| `queue_necessary_classifications` | `sde_collections/models/collection.py:692` | The inference on/off switch; its `else` branch already calls migration directly. | +| `inference/signals.py:5` | `post_migrate` → `PeriodicTask` | The exact pattern to copy for the new `poll_scrape_jobs` schedule. There is **no `CELERY_BEAT_SCHEDULE`** in this repo — all schedules are DB rows. | +| `send_detailed_import_notification` | `sde_collections/utils/slack_utils.py:62` | Already written, **never called**. A ready-made ingest-summary Slack hook. | +| `STATUS_CHANGE_NOTIFICATIONS` | `sde_collections/utils/slack_utils.py:12` | `(old, new)` → message map. Already covers `QC:Perfect→Prod:Perfect` and `QC:Minor→Prod:Minor`. | +| `s3_keys_for_collection` | `sde-crawl4ai-scraper-v1/sde_crawler/job.py:65` | Authoritative S3 key layout — mirror it, don't reinvent it. | +| `JOB_DEFAULTS` / `merge_job` | `sde-crawl4ai-scraper-v1/sde_crawler/job.py` | Job JSON contract. `None` values are skipped, so COSMOS should emit only non-null overrides. | + +--- + +## Repo structure after this work + +``` +COSMOS/ +├── WORKFLOW.md ← tracked (was untracked) +├── WORKFLOW_DIAGRAM.png ← tracked +├── ecr.override.yml ★ P9 +├── scripts/deploy.sh ★ P9 +├── .github/workflows/ +│ ├── ci.yml ★ P9 (replaces run_full_test_suite.yml) +│ ├── deploy-staging.yml ★ P9 +│ ├── deploy-production.yml ★ P9 +│ ├── rollback.yml ★ P9 +│ └── secret-scan-history.yml ★ P9 +├── sde_collections/ +│ ├── DEPLOYMENT.md ← tracked +│ ├── apps.py ✎ add ready() → import signals +│ ├── signals.py ★ P4 post_migrate → poll_scrape_jobs PeriodicTask +│ ├── tasks.py ✎ scrape dispatch / poll / ingest; Sinequa tasks removed +│ ├── sinequa_api.py ✖ DELETED (P6) +│ ├── models/ +│ │ ├── collection.py ✎ triggers rewired, Sinequa methods removed +│ │ ├── collection_choice_fields.py ✎ statuses 21–26 +│ │ ├── scraper_config.py ★ P3 ScraperConfigOverride + ScrapeDispatch +│ │ ├── indexing.py ★ P7 IndexDispatch +│ │ └── delta_url.py ✔ UNCHANGED (hard constraint) +│ ├── scraping/ ★ P3/P4 new subpackage +│ │ ├── job_builder.py ★ build_job_json(collection) -> dict +│ │ ├── ssm_dispatch.py ★ send_job_to_crawler(collection) +│ │ └── s3_results.py ★ fetch summary + documents from S3 +│ ├── indexing/ ★ P7 new subpackage (mirrors sde-api-scrapers W4) +│ │ ├── export.py ★ export_curated_to_s3(collection, target, run_id) +│ │ └── dispatch.py ★ run_index_task(collection, target, run_id) +│ ├── utils/ +│ │ ├── aws.py ★ P0 get_boto3_session() — shared credential chain +│ │ ├── slack_utils.py ✎ new transitions +│ │ ├── github_helper.py ✖ DELETED (P6) +│ │ ├── bulk_github_push.py ✖ DELETED (P6) +│ │ └── health_check.py ✖ DELETED (P6) +│ └── management/commands/ +│ ├── preflight_aws.py ★ P9 +│ ├── validate_deploy_env.py ★ P9 +│ ├── dispatch_scrape.py ★ P3 manual re-dispatch +│ ├── ingest_scrape_results.py ★ P4 manual ingest +│ ├── import_from_sinequa.py ✖ DELETED (P6) +│ ├── push_to_github.py ✖ DELETED (P6) +│ ├── sync_all_with_github.py ✖ DELETED (P6) +│ ├── load_urls_from_api.py ✖ DELETED (P6) +│ └── generate_configs.py ✖ DELETED (P6 — already dead code) +├── config_generation/ ✖ DELETED ENTIRELY (P6) +├── default_scraper.xml ✖ DELETED (P6) +├── inference/ ✎ P2 gated off, NOT deleted +└── config/settings/base.py ✎ new AWS / crawler / flag settings +``` + +`★` new · `✎` modified · `✖` deleted · `✔` explicitly untouched + +--- + +## Phase 0 — Foundations: settings, AWS session helper, doc tracking + +**Goal:** land the shared plumbing every later phase needs, with zero behavior change. + +### Files + +| File | Change | +|---|---| +| `config/settings/base.py` | Add the settings block below (near `SLACK_WEBHOOK_URL`, ~L346) | +| `sde_collections/utils/aws.py` | **New.** `get_boto3_session()` | +| `.env_sample`, `.envs/.local/.django` | Document the new vars | +| `requirements/base.txt` | No change yet (boto3 already pinned at `1.34.31`) | +| `git add` the untracked docs | Bring `WORKFLOW.md`, `WORKFLOW_DIAGRAM.png`, `sde_collections/DEPLOYMENT.md` under version control | + +### New settings + +```python +# --- SDE curation pipeline --- +AWS_REGION = env("AWS_REGION", default="us-east-1") +SDE_S3_BUCKET = env("SDE_S3_BUCKET", default="") # crawler output bucket +CRAWLER_INSTANCE_ID = env("CRAWLER_INSTANCE_ID", default="") # i-0b6a61d95888886f4 on dev +CRAWLER_INBOX_PATH = env("CRAWLER_INBOX_PATH", default="/opt/sde-crawler/jobs/incoming") +SCRAPE_POLL_ENABLED = env.bool("SCRAPE_POLL_ENABLED", default=False) +INFERENCE_ENABLED = env.bool("INFERENCE_ENABLED", default=False) +# pipeline-scoped credentials for local dev ONLY; blank in AWS (instance role takes over) +SDE_AWS_ACCESS_KEY_ID = env("SDE_AWS_ACCESS_KEY_ID", default="") +SDE_AWS_SECRET_ACCESS_KEY = env("SDE_AWS_SECRET_ACCESS_KEY", default="") +# COSMOS never talks to OpenSearch or SageMaker: chunk/vectorize/index AND the QC validation +# report are produced by the WEB_COSMOS task in sde-api-scrapers (branch web-indexing), which +# holds the AOSS credentials. The P7 dispatch/poll settings — SDE_INDEX_BUCKET (distinct from +# SDE_S3_BUCKET), INDEXING_ECS_CLUSTER, INDEXING_TASK_FAMILY, INDEXING_DISPATCH_ROLE_ARN, +# INDEX_POLL_ENABLED — land with P7, all with defaults so config.settings.test keeps booting. +``` + +> **Naming note:** existing settings use the `DJANGO_AWS_*` prefix for the *static assets* +> bucket/credentials. `SDE_S3_BUCKET` and `SDE_AWS_*` are deliberately distinct names — do not +> merge the two. In particular the helper below must **not** read `settings.AWS_ACCESS_KEY_ID`: +> that is the django-storages static-assets credential, it is defined only in `local.py:63` and +> `production.py:48` (not `base.py`), so referencing it under `test.py` raises `AttributeError` — +> and it is the wrong credential scope anyway. + +### `sde_collections/utils/aws.py` + +Existing COSMOS AWS code passes **static access keys** (`tasks.py:143`, `health_check.py:199`), +but `DEPLOYMENT.md` assumes the deployed host's **instance role**. Reconcile with one helper: + +```python +def get_boto3_session(): + """Default credential chain (instance role in AWS); explicit SDE keys only if set (local dev).""" + if settings.SDE_AWS_ACCESS_KEY_ID and settings.SDE_AWS_SECRET_ACCESS_KEY: + return boto3.Session( + aws_access_key_id=settings.SDE_AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.SDE_AWS_SECRET_ACCESS_KEY, + region_name=settings.AWS_REGION, + ) + return boto3.Session(region_name=settings.AWS_REGION) +``` + +All new SSM/S3/AOSS/SageMaker code uses this. Do not add new static-key call sites. + +### Validation + +```bash +docker-compose -f local.yml build +docker-compose -f local.yml up -d +docker-compose -f local.yml run --rm django python manage.py check +docker-compose -f local.yml run --rm django pytest # full suite must still pass +``` + +New test `sde_collections/tests/test_aws_utils.py`: asserts the session falls back to the default +chain when the `SDE_AWS_*` keys are blank (their `base.py` default, so no `override_settings` +gymnastics needed), and uses explicit keys when set (mock `boto3.Session`). + +### Done when +- [x] Settings added; `manage.py check` clean with all new vars unset (defaults hold) +- [x] `get_boto3_session()` exists with tests +- [x] Four rewiring docs committed +- [x] Full existing test suite green + +--- + +## Phase 1 — Workflow statuses 21–26, UI colour maps, Slack transitions + +**Goal:** make the new statuses selectable and renderable everywhere. **No triggers yet.** + +### Files + +| File | Change | +|---|---| +| `sde_collections/models/collection_choice_fields.py:80` | Add the six members below to `WorkflowStatusChoices` | +| `sde_collections/models/collection.py:333` | Add keys `24, 25, 26` to `Collection.workflow_status_button_color` | +| `sde_collections/models/collection.py:797` | Add keys `24, 25, 26` to `WorkflowHistory.workflow_status_button_color` | +| `sde_indexing_helper/static/js/collection_list.js:317` | Extend `color_choices` — it currently covers **only 1–16** | +| `sde_collections/utils/slack_utils.py:12` | Add the new transitions | +| `sde_collections/models/README_STATUS_TRIGGERS.md` | Document the new statuses | +| new migration `0078_*` | Choices-only `AlterField` (additive, reversible) | + +```python +SCRAPING_SUCCESSFUL = 21, "Scraping Successful" +TEST_INDEXING = 22, "Test Indexing" +SCRAPING_FAILED = 23, "Scraping Failed" +INDEXING_FAILED_ON_TEST = 24, "Indexing Failed on Test" +INDEXING_FAILED_ON_PROD = 25, "Indexing Failed on Prod" +PRODUCTION_INDEXING = 26, "Production Indexing" +``` + +> **Latent bug this phase must fix:** both Python colour maps do a bare +> `color_choices[self.workflow_status]` — an unmapped status raises `KeyError` and breaks the +> collection list *and* detail pages. Keys 21–23 already exist; 24–26 do not. Change the lookups to +> `color_choices.get(self.workflow_status, "btn-light")` so this class of failure cannot recur. +> The JS map is worse: statuses 17–20 already render colourless today. Fill 17–26 in the same pass. +> +> While in there: key `23` (Scraping Failed) is pre-provisioned as `btn-light` in both Python maps +> (`collection.py:357,821`) — change it to `btn-danger`, and use `btn-danger` for 24/25 too. +> Failure statuses must not render as neutral. + +New Slack transitions to add: + +| Transition | Message | +|---|---| +| `READY_FOR_ENGINEERING → SCRAPING_SUCCESSFUL` | scrape finished, counts included | +| `READY_FOR_ENGINEERING → SCRAPING_FAILED` | alert, mention devs | +| `CURATED → INDEXING_FAILED_ON_TEST` | alert | +| `* → INDEXING_FAILED_ON_PROD` | alert, mention devs | + +### Validation + +```bash +docker-compose -f local.yml run --rm django python manage.py makemigrations +docker-compose -f local.yml run --rm django python manage.py migrate +docker-compose -f local.yml run --rm django pytest sde_collections/tests/ +``` + +New tests in `sde_collections/tests/test_workflow_status_triggers.py`: +- Every `WorkflowStatusChoices` member resolves a colour on `Collection` **and** `WorkflowHistory` + (parametrised — this is the regression guard for the `KeyError`). +- Setting each new status writes a `WorkflowHistory` row. + +### Manual verification +1. `docker-compose -f local.yml up`, open `http://localhost:8001/`. +2. The per-row workflow dropdown lists all 26 statuses; the filter panel shows them. +3. Select **Scraping Successful** on a collection — the button re-colours, no console error, + and the detail page **Workflow History** tab shows the transition. + +### Done when +- [x] Six statuses added, migration applied +- [x] Both Python colour maps use `.get(...)` with a default; JS map covers 1–26 +- [x] Slack map has the four new transitions +- [x] Parametrised colour test passes for every enum member +- [x] Dropdowns/filters render all statuses with no JS console errors + +--- + +## Phase 2 — Disable the inference pipeline (do not delete) + +**Goal:** path of least resistance — keep the functionality, just stop it running. + +### The trap + +`queue_necessary_classifications()` (`collection.py:692`) routes **three hard-coded collections** +(`imagine_the_universe`, `physics_of_the_cosmos`, `stsci_space_telescope_science_institute`) +through `InferenceJob`; only the `else` branch calls the migration task directly. Disabling only +the beat schedule would leave those three collections with a queued job that never runs, so they +**never reach `Ready for Curation`**. The flag must short-circuit the branch itself. + +### Files + +| File | Change | +|---|---| +| `sde_collections/models/collection.py:692` | Early return in `queue_necessary_classifications()` when `not settings.INFERENCE_ENABLED` → call `migrate_dump_to_delta_and_handle_status_transistions.delay(self.id)` and return | +| `inference/signals.py:33,49` | Pass `enabled=settings.INFERENCE_ENABLED` on `PeriodicTask.objects.create(...)`, and set `.enabled` in both update branches | +| `inference/tasks.py:8` | Belt-and-braces early return when the flag is off | + +> `inference/signals.py` currently re-asserts `crontab` and `task` on **every** `post_migrate`, so +> disabling the rows by hand in the admin does not survive a deploy. Setting `enabled` in the +> signal is what makes the disable durable — exactly the failure mode `DEPLOYMENT.md` warns about. + +Leave `inference` in `INSTALLED_APPS`, leave models and migrations alone — +`candidate_url.py:70` (`inferenced_by`), the paired ML/manual fields, and +`test_import_fulltexts.py` all still import from it. + +### Validation + +```bash +docker-compose -f local.yml run --rm django pytest inference/ sde_collections/tests/ +docker-compose -f local.yml run --rm django python manage.py migrate # re-run: rows stay disabled +docker-compose -f local.yml logs -f celerybeat # no inference queue ticks +``` + +New tests: +- With `INFERENCE_ENABLED=False`, a collection whose `config_folder` is in the TDAMM list still + calls the migration task and creates **no** `InferenceJob`. *(This is the regression guard.)* +- With the flag on, the TDAMM path still creates an `InferenceJob`. +- After `post_migrate`, both `PeriodicTask` rows have `enabled == INFERENCE_ENABLED`. + +> **Test fidelity:** the existing suite patches `queue_necessary_classifications` wholesale +> (`test_workflow_status_triggers.py:428`). The two flag tests above must call the **real** method +> under `override_settings(INFERENCE_ENABLED=...)` (patching only `migrate_dump_to_delta_…​.delay`), +> or they guard nothing. + +### Manual verification +`shell_plus` → `PeriodicTask.objects.filter(task__startswith="inference").values("name","enabled")` +→ both `False`. Confirm `celerybeat` logs show no inference ticks over ~10 min. + +### Done when +- [x] Flag added and honoured at all three sites +- [x] TDAMM collections still migrate to `Ready for Curation` with inference off +- [x] `PeriodicTask.enabled` survives a re-run of `migrate` + +--- + +## Phase 3 — Scrape dispatch: overrides model + job JSON + SSM + +**Goal:** `Ready for Engineering` writes a job JSON into the crawler's inbox via SSM. +(WORKFLOW.md steps 5–7.) + +### `collection_id` decision + +The scraper derives `collection_id` from the seed host when absent, and uses it for both the output +filename and the S3 key. COSMOS will **always send `config_folder`** as `collection_id`: +it is unique, `editable=False`, already the AOSS `collection_key`, and stable across renames. +Do not use the numeric PK. + +### Files + +| File | Change | +|---|---| +| `sde_collections/models/scraper_config.py` | **New.** `ScraperConfigOverride` **and `ScrapeDispatch`** (below) | +| `sde_collections/models/__init__.py` | Export both | +| `sde_collections/admin.py` | Register both (WORKFLOW.md step 6: curators edit overrides in the admin console; `ScrapeDispatch` read-only for debugging) | +| `sde_collections/scraping/job_builder.py` | **New.** `build_job_json(collection) -> dict` | +| `sde_collections/scraping/ssm_dispatch.py` | **New.** `send_job_to_crawler(collection) -> str` (SSM command id) | +| `sde_collections/tasks.py` | **New task** `dispatch_scrape_job(collection_id)` — records a `ScrapeDispatch` row on send | +| `sde_collections/models/collection.py:887` | `READY_FOR_ENGINEERING` branch: replace `create_scraper_config`/`create_scraper_job` with `dispatch_scrape_job.delay(instance.id)`. **Also** add `REINDEXING_NEEDED_ON_DEV → dispatch_scrape_job.delay(instance.id)` in the `reindexing_status` block (per the reindexing decision above — this replaces the engineer manually re-running a Sinequa job) | +| `sde_collections/management/commands/dispatch_scrape.py` | **New.** Manual re-dispatch | +| new migration `0079_*` | Creates `ScraperConfigOverride` + `ScrapeDispatch` (additive) | + +```python +class ScraperConfigOverride(models.Model): + """Per-collection overrides merged onto the crawler's own defaults. + All fields nullable: only non-null values are emitted into the job JSON, + because crawl4ai's merge_job() skips None.""" + collection = models.OneToOneField(Collection, on_delete=models.CASCADE, + related_name="scraper_config") + max_pages = models.PositiveIntegerField(null=True, blank=True) # cap 100_000 + depth_limit = models.PositiveIntegerField(null=True, blank=True) + delay = models.FloatField(null=True, blank=True) # default 0.25 + concurrent_requests = models.PositiveSmallIntegerField(null=True, blank=True) + obey_robots = models.BooleanField(null=True, blank=True) + include_subdomains = models.BooleanField(null=True, blank=True) + + +class ScrapeDispatch(models.Model): + """One row per SSM dispatch. Two jobs: (1) give the poller a freshness reference — + S3 results older than dispatched_at belong to a previous run and must be ignored; + (2) give the stall timeout a start time. Never deleted; latest row per collection wins.""" + collection = models.ForeignKey(Collection, on_delete=models.CASCADE, + related_name="scrape_dispatches") + dispatched_at = models.DateTimeField(auto_now_add=True) + ssm_command_id = models.CharField(max_length=64) +``` + +`build_job_json` emits `{"seed": collection.url, "collection_id": collection.config_folder}` plus +**only the non-null** override fields. Validate `max_pages <= 100_000` before dispatch — +the crawler raises `ValueError` above the cap and the job would land in `jobs/failed/`. + +`send_job_to_crawler` mirrors `scripts/drop_job.sh`: `ssm.send_command`, +`DocumentName="AWS-RunShellScript"`, writing to +`{CRAWLER_INBOX_PATH}/{config_folder}.json` and `chown ec2-user`. JSON must be shell-quoted with +`shlex.quote` — seed URLs contain characters that will otherwise break the heredoc. + +### Validation + +```bash +docker-compose -f local.yml run --rm django python manage.py makemigrations +docker-compose -f local.yml run --rm django python manage.py migrate +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_scrape_dispatch.py +``` + +New `sde_collections/tests/test_scrape_dispatch.py` (all AWS mocked): +- `build_job_json` with no overrides → exactly `{seed, collection_id}`. +- With `max_pages=5000, delay=None` → `delay` **absent**, `max_pages` present. +- `max_pages=200_000` raises before any SSM call. +- Status → `READY_FOR_ENGINEERING` calls `dispatch_scrape_job.delay` once (patch at + `sde_collections.tasks.dispatch_scrape_job.delay`). +- `reindexing_status → REINDEXING_NEEDED_ON_DEV` also calls `dispatch_scrape_job.delay` once. +- A successful dispatch creates a `ScrapeDispatch` row carrying the SSM command id. +- SSM failure sets `SCRAPING_FAILED`, creates **no** `ScrapeDispatch` row, and does not raise out + of the task. + +### Manual verification (real dev AWS, `sde-dev` profile) +```bash +docker-compose -f local.yml run --rm django python manage.py dispatch_scrape --collection +aws ssm send-command --instance-ids i-0b6a61d95888886f4 \ + --document-name AWS-RunShellScript \ + --parameters 'commands=["ls -la /opt/sde-crawler/jobs/incoming/"]' --profile sde-dev +# then confirm the watcher picked it up: +# tail /opt/sde-crawler/logs/watch.log and ls /opt/sde-crawler/jobs/{done,failed}/ +``` + +### Done when +- [x] `ScraperConfigOverride` + `ScrapeDispatch` created, admin-visible, migration applied +- [x] Job JSON omits null overrides and rejects `max_pages > 100_000` +- [x] `READY_FOR_ENGINEERING` **and** `REINDEXING_NEEDED_ON_DEV` dispatch via SSM; Sinequa scraper-config calls gone from those branches +- [x] Every dispatch records `dispatched_at` + `ssm_command_id` +- [x] A real job JSON lands in the dev crawler inbox and moves to `jobs/done/` + +--- + +## Phase 4 — Poll for results and ingest S3 → `DumpUrl` + +**Goal:** WORKFLOW.md steps 9–11. This is where `fetch_full_text` is functionally replaced. + +### Completion contract + +The crawler writes **no status file**. The reliable signal is S3, **filtered for freshness** — +S3 objects persist between runs, so every check below only counts an object whose `LastModified` +is **after the collection's latest `ScrapeDispatch.dispatched_at`** (P3). Without this, any +re-dispatch would instantly "complete" against the previous run's output. + +| Object | Meaning | +|---|---| +| `failure_logs/{cid}_failures_summary.json` | Written **only** at the end of a completed run — this is the completion marker | +| `scraped_collections/{cid}.json` | The documents array | +| `failure_logs/{cid}_failures.jsonl` | Per-URL failures | + +Fresh summary + `documents_scraped > 0` → **`SCRAPING_SUCCESSFUL`**. +Fresh summary + `documents_scraped == 0` → **`SCRAPING_FAILED`** (a zero-page crawl otherwise +"succeeds" and would silently produce an empty collection). +No fresh summary after the stall timeout (measured from `dispatched_at`) → **`SCRAPING_FAILED`** +(died mid-run, or the job never started). + +Document shape is exactly seven fields — `{url, title, full_text, content_type, seed, host, depth}` — +which maps cleanly onto `DumpUrl(url=, scraped_title=, scraped_text=)`, the same contract +`fetch_full_text` already satisfies. + +### Files + +| File | Change | +|---|---| +| `sde_collections/scraping/s3_results.py` | **New.** `fetch_summary(cid)`, `fetch_documents(cid)`, `results_ready(cid)` | +| `sde_collections/tasks.py` | **New tasks** `poll_scrape_jobs()` and `ingest_scraped_collection(collection_id)` | +| `sde_collections/signals.py` | **New.** `post_migrate` → `poll_scrape_jobs` `PeriodicTask` (every 5 min), `enabled=settings.SCRAPE_POLL_ENABLED` | +| `sde_collections/apps.py` | Add `ready()` → `import signals` (currently 6 lines, no `ready()`) | +| `sde_collections/management/commands/ingest_scrape_results.py` | **New.** Manual ingest | +| `sde_collections/utils/slack_utils.py` | Wire the **already-written, never-called** `send_detailed_import_notification` into the ingest task | + +`poll_scrape_jobs` scans collections in `READY_FOR_ENGINEERING` **or** `ENGINEERING_IN_PROGRESS` +(engineers flip to the latter while a crawl runs — a collection must not strand there), **plus** +collections with `reindexing_status == REINDEXING_NEEDED_ON_DEV` (the re-scrape path from P3), +checks S3 with the freshness rule above, and enqueues `ingest_scraped_collection`. + +`ingest_scraped_collection` claims the collection **first**, then mirrors `fetch_full_text`'s body: + +1. **Atomic claim** — the status transition is the lock, executed as a compare-and-swap: + `Collection.objects.filter(id=..., workflow_status__in=[READY_FOR_ENGINEERING, + ENGINEERING_IN_PROGRESS]).update(workflow_status=SCRAPING_SUCCESSFUL)`; if it returns 0, another + ingest already claimed it — exit. (For the re-scrape path the CAS is on + `reindexing_status: REINDEXING_NEEDED_ON_DEV → REINDEXING_FINISHED_ON_DEV` instead.) + Claiming *before* the write matters: ingest can outrun the 5-minute poll, and `BaseUrl.url` is + globally `unique=True`, so two concurrent ingests would die on `IntegrityError` mid-write. +2. Delete existing `DumpUrl`s → `bulk_create` in batches. +3. `collection.queue_necessary_classifications()` (which, with inference off from P2, calls + `migrate_dump_to_delta_and_handle_status_transistions` → `READY_FOR_CURATION`; on the re-scrape + path the migrate task's existing reindexing transition promotes to + `REINDEXING_READY_FOR_CURATION`). +4. On ingest failure after a successful claim, set `SCRAPING_FAILED` — never leave a claimed + collection stuck in `SCRAPING_SUCCESSFUL` with no `DumpUrl`s. + +`tasks.py:216` `pre_workflow_statuses` must gain `SCRAPING_SUCCESSFUL` so the migration task +promotes to `READY_FOR_CURATION` from the new status. + +> Replay stays safe: a manual re-run via `ingest_scrape_results` skips the CAS (explicit operator +> intent) but still deletes existing `DumpUrl`s first, so it is idempotent. + +### Validation + +```bash +docker-compose -f local.yml run --rm django python manage.py migrate # creates the beat row +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_scrape_ingest.py +docker-compose -f local.yml logs -f celeryworker +``` + +New `sde_collections/tests/test_scrape_ingest.py` — S3 mocked with a fixture built from the real +7-field document shape: +- Happy path: N documents → N `DumpUrl`s with `scraped_title`/`scraped_text` populated → + status `SCRAPING_SUCCESSFUL`. +- `documents_scraped == 0` → `SCRAPING_FAILED`, no `DumpUrl`s. +- Missing summary → collection stays `READY_FOR_ENGINEERING`, nothing enqueued. +- **Stale results:** summary `LastModified` **before** the latest `ScrapeDispatch.dispatched_at` → + treated as absent; nothing enqueued. *(Re-dispatch regression guard.)* +- **Stall timeout:** no fresh summary and `dispatched_at` older than the timeout → `SCRAPING_FAILED`. +- **Concurrent claim:** second `ingest_scraped_collection` on an already-claimed collection exits + without touching `DumpUrl`s (CAS returns 0). +- Re-running manual ingest twice yields N (not 2N) `DumpUrl`s — idempotency. +- Re-scrape path: `reindexing_status == REINDEXING_NEEDED_ON_DEV` + fresh results → ingest → + `REINDEXING_READY_FOR_CURATION`. +- With inference disabled, ingest reaches `READY_FOR_CURATION`. *(End-to-end P2+P4 check.)* + +### Manual verification (real dev AWS) +```bash +aws s3 ls s3://sdecrawlerstack-crawlbucket0d63eba8-lhkxqnh8ophy/scraped_collections/ --profile sde-dev +docker-compose -f local.yml run --rm django python manage.py ingest_scrape_results --collection +docker-compose -f local.yml run --rm django python manage.py shell_plus +>>> c = Collection.objects.get(config_folder="") +>>> c.dump_urls.count(), c.delta_urls.count(), c.get_workflow_status_display() +``` +Expect dump count to match `documents_scraped` in the summary, deltas created, status +`Ready for Curation`. + +### Done when +- [x] S3 completion contract implemented, including the zero-document failure case **and the + `dispatched_at` freshness rule** +- [x] `poll_scrape_jobs` beat row created via `post_migrate`, gated on `SCRAPE_POLL_ENABLED` +- [x] Ingest claims via CAS before writing, is idempotent, and populates `DumpUrl` without touching the model +- [x] Re-scrape path (`REINDEXING_NEEDED_ON_DEV`) polls and ingests end to end +- [x] A real dev collection goes seed URL → `Ready for Curation` end to end +- [x] Slack posts an ingest summary *(wired in the migrate task; was blocked by the Sinequa + `READY_FOR_CURATION → create_indexer_config` branch raising mid-task — P5 removed that + branch, and the P4 end-to-end test now runs unpatched, asserting the notification + counts)* + +--- + +## Phase 5 — Curation triggers and the indexing hand-off seam + +**Goal:** WORKFLOW.md steps 12–18, and a clean boundary for the deferred indexing work. + +### Files + +| File | Change | +|---|---| +| `sde_collections/models/collection.py:871` | Rewrite `create_configs_on_status_change`: drop the `READY_FOR_CURATION` indexer-config branch and the `INDEXING_FINISHED_ON_DEV` branch; keep `CURATED → promote_to_curated()`; replace `QC_PERFECT/QC_MINOR → add_to_public_query()` with the prod-indexing hand-off. **In the `reindexing_status` block:** drop `REINDEXING_FINISHED_ON_DEV → fetch_full_text` (the P4 ingest sets that status itself — a trigger here would double-fire); keep `REINDEXING_CURATED → promote_to_curated()`; `REINDEXING_NEEDED_ON_DEV → dispatch_scrape_job` was already added in P3 | +| `sde_collections/tasks.py` | **New stub tasks** `index_collection_to_test(collection_id)` and `index_collection_to_prod(collection_id)` | +| `sde_collections/models/collection.py:733` | Move the Slack block out of `save()` into the `post_save` signal | + +The rewired dispatcher: + +``` +workflow_status: + READY_FOR_ENGINEERING -> dispatch_scrape_job.delay(id) (from P3) + CURATED -> promote_to_curated(); index_collection_to_test.delay(id) + QC_PERFECT / QC_MINOR -> index_collection_to_prod.delay(id) + +reindexing_status: + REINDEXING_NEEDED_ON_DEV -> dispatch_scrape_job.delay(id) (from P3) + REINDEXING_CURATED -> promote_to_curated() +``` + +> **Rename the dispatcher.** `create_configs_on_status_change` no longer creates configs. +> Rename to `handle_workflow_status_change` and update +> `sde_collections/models/README_STATUS_TRIGGERS.md`. + +> **Move the Slack call.** It currently sits in `Collection.save()` **before** `super().save()` +> and issues its own extra `Collection.objects.get()` query — so a message can be sent for a save +> that then fails. Moving it into the existing `post_save` receiver (which already has +> `old_workflow_status`) removes the extra query and the false-positive notification. + +In this phase `index_collection_to_test` / `index_collection_to_prod` are **stubs**: they set +`TEST_INDEXING` / `PRODUCTION_INDEXING`, log, and return. Statuses 22/24/25/26 are therefore +defined and wired but not yet driven to completion. + +These stubs are the **event-trigger seam for the `WEB_COSMOS` pipeline** (built in +`sde-api-scrapers`, branch `web-indexing` — see the decisions table). Phase 7 turns them into +export-then-dispatch tasks: mint a `run_id`, export the curated set to S3 +(`documents.jsonl` + `manifest.json`, manifest last), assume the dispatch role, and `ecs:RunTask` +with a command override (`python3 api_scraper.py --source WEB_COSMOS --collection --target test|prod +--run-id `) — they never index in-process. This is what replaces the api pipeline's +EventBridge schedule: indexing runs as soon as a collection is curated, not on a timer. + +### Validation + +```bash +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_workflow_status_triggers.py +``` + +Rewrite that file for the new trigger table — the existing tests assert Sinequa behaviour and +**will fail by design**. Cover: `CURATED` promotes **and** enqueues test indexing; `QC_*` enqueues +prod indexing; `REINDEXING_FINISHED_ON_DEV` triggers **nothing** (P4's ingest owns that +transition); `REINDEXING_CURATED` still promotes; no Sinequa method is called on any transition; +the re-entrancy guard (`_handling_status_change`) still prevents recursion. + +### Manual verification +Drive a collection through `Ready for Curation → Curation in Progress → Curated` in the UI. +Confirm `CuratedUrl`s appear, `DeltaUrl`s are cleared, status lands on `Test Indexing`, +and Slack posts the curation message. + +### Done when +- [x] Dispatcher renamed and rewired; no Sinequa calls remain in the trigger path +- [x] Slack moved into `post_save` +- [x] Indexing stubs enqueue and set the in-flight statuses +- [x] `test_workflow_status_triggers.py` rewritten and green + +--- + +## Phase 6 — Delete Sinequa + +**Goal:** remove Sinequa entirely. Nothing in the pipeline path depends on it after +Phase 5, so this is now a pure removal. + +### Delete outright + +| Category | Paths | +|---|---| +| API client | `sde_collections/sinequa_api.py` | +| Config generation | `config_generation/` (**entire directory**, incl. `xmls/`, `plugins/`, `tests/`) | +| XML templates | `default_scraper.xml`, `sde_collections/xml_templates/` | +| GitHub push | `sde_collections/utils/github_helper.py`, `sde_collections/utils/bulk_github_push.py` | +| Health check | `sde_collections/utils/health_check.py` (built on `_get_data_to_import`) | +| Commands | `import_from_sinequa.py`, `push_to_github.py`, `sync_all_with_github.py`, `load_urls_from_api.py`, `generate_configs.py` | +| Tests | `sde_collections/tests/test_sinequa_api.py`, `sde_collections/tests.py` (legacy), `config_generation/tests/` | +| Docs | `docs/documentation/sinequa_api.rst` + its `docs/index.rst` toctree entry | +| Scripts | `scripts/push_curated_collections_to_github.py`, `scripts/update_has_sinequa_config.py` | + +### Edit + +| File | Removal | +|---|---| +| `collection.py` | `_scraper_config_path`, `_indexer_config_path`, `_indexer_job_path`, `_scraper_job_path`, `add_to_public_query`, `server_url_prod`, `server_url_secret_prod`, `_write_to_github`, `create_scraper_config`, `create_indexer_config`, `create_scraper_job`, `create_indexer_job`, `update_config_xml`, `import_metadata_from_sinequa_config`, `sinequa_configuration`, `_process_exclude_list`/`_include`/`_title`/`_document_type`, and the `XmlEditor`/`GitHubHandler` imports | +| `tasks.py` | `_get_data_to_import`, `import_candidate_urls_from_api`, `push_to_github_task`, `pull_latest_collection_metadata_from_github`, **`fetch_full_text`** (replaced in P4 — safe to delete only because P5 already removed both of its trigger sites, `INDEXING_FINISHED_ON_DEV` and `REINDEXING_FINISHED_ON_DEV`; verify no import of it remains in `collection.py`) | +| `views.py` | `PushToGithubView` (L509), `IndexingInstructionsView` (L517), the health-check/consolidation view; plus their `urls.py` routes | +| `config/settings/base.py` | `GITHUB_ACCESS_TOKEN`, `SINEQUA_CONFIGS_*`, `XLI_*`, `LRM_DEV_*`, `LRM_QA_*` — note these are all `env(...)` with **no defaults** (`base.py:341–354`), so today the app cannot even boot without dummy Sinequa secrets. Removing them is what makes clean-host deploys possible, which is why **P6 must land before P9's first deploy to a fresh host** | +| `requirements/base.txt` | `PyGithub`, `xmltodict`; check `lxml` for other consumers before dropping | +| Templates / JS | Sinequa config link in `collection_detail.html`; matching handler in `static/js/project.js` | + +### Keep (deliberately) + +- `SourceChoices.ONLY_IN_SINEQUA_CONFIGS` — a historical **data** value on existing rows. + Removing it would need a data migration; leave it. +- `Collection.config_folder` — now the crawler `collection_id` and the AOSS `collection_key`. +- `scraper/` (legacy Scrapy project) — **out of scope**; unrelated to the Sinequa retirement. + Flag for a separate cleanup. + +> `utils/health_check.py` calls `WorkflowStatusChoices.get_status_string()` (L90, L119), which does +> not exist on that enum — a latent `AttributeError`. Deleting the file resolves it. If any part of +> health-check is worth keeping, the method must be added to the enum first. + +### Validation + +```bash +docker-compose -f local.yml run --rm django python manage.py check +docker-compose -f local.yml run --rm django pytest +grep -ri "sinequa\|XmlEditor\|GitHubHandler\|PyGithub" --include='*.py' --include='*.html' --include='*.js' . +``` +The grep should return only `SourceChoices.ONLY_IN_SINEQUA_CONFIGS` and changelog/history text. + +Unset every deleted env var in `.envs/.local/.django` **before** running — `config/settings/test.py` +inherits `base.py`, so a lingering required var would mask a missed reference. + +### Done when +- [x] All listed files deleted; `manage.py check` clean +- [x] Full test suite green with the Sinequa env vars **unset** +- [x] Grep is clean apart from the retained enum member (plus historical migration text and + comments, per the "changelog/history" allowance) +- [x] `requirements/base.txt` pruned; image rebuilds + +--- + +## Phase 7 — Indexing hand-off: export, dispatch, poll (= `sde-api-scrapers` Phase W4) + +**No longer "deferred until the pipeline exists" — the pipeline exists and is deployed to dev.** +It landed as a `WEB_COSMOS` source in **`sde-api-scrapers`** (built on branch `web-indexing`, +**merged to `develop` in PR #47 @ `c2aebad` and deployed to dev 2026-08-20** — NS.2; its phases W0 +foundations, W1 shared-component parameterization, W2 web pipeline, W3 infrastructure, and W5 tests +are code-complete: 316 offline tests green — but **no run has executed against real AWS yet**, and +all runtime defaults there target the disposable **`sde-web-subset`** +index — a scratch subset of live `sde-web`, which replaced `sde-web-copy` on 2026-08-19 (`10975ee`) — until an explicit cutover). What remains here is exactly that repo's **Phase W4 — the +COSMOS side**: export, dispatch, poll, Slack. Authoritative docs on that branch: `DESIGN.md`, +`Web Indexing - Task Plan & Tracking.md`, `open_questions.md` (decision record), +`FINDING_id_scheme_collision.md`. + +**Phase 5 landed (`9b3df519`) and this phase is committed (`6fa19843`, dispatch fix `20c919ed`).** +Everything indexer-side is independent of COSMOS and is exercised from its own CLI against +hand-written exports; only the closed loop (their E2E.10) is still open — see "Cross-repo status". + +### The contract (fixed by the built indexer — do not re-negotiate silently) + +1. **Export (COSMOS → S3).** Write to the dedicated bucket **`sde-cosmos-indexing-{env}`** + (owned by the indexer stacks; **not** the crawler's `SDE_S3_BUCKET`): + + ``` + curated_collections/{config_folder}/{run_id}/documents.jsonl + curated_collections/{config_folder}/{run_id}/manifest.json ← written LAST = "export complete" + ``` + + `run_id` is minted by COSMOS and threaded through every artifact. Each JSONL line: + `{url, title, full_text, document_type, division, tdamm_tag, is_metadata_viewer}` — per-URL + `division`/`document_type` only when they differ from the collection default. The manifest: + `{schema_version, run_id, collection_key (= config_folder), collection_name, division, target, + document_count, exported_at, cosmos_workflow_status}`. The indexer verifies line count against + `document_count` and skips deletions on mismatch, so the count must be exact. + +2. **Dispatch.** Assume **`CosmosIndexingDispatchRole-{env}`** (same-account in dev — see + "Cross-repo status" below; the assume-role indirection is kept so test/prod can be genuinely + cross-account later), then `ecs:RunTask` + with a **command override**: `python3 api_scraper.py --source WEB_COSMOS --collection {config_folder} + --target test|prod --run-id {run_id}`. (Not container-env overrides — the earlier + `COLLECTION_ID`/`TARGET` sketch is superseded. Target→endpoint resolution is tier-capped on the + indexer side, so a dev dispatch can never reach prod AOSS.) + +3. **Completion (S3 poll — not `ecs.describe_tasks`).** The indexer writes + `index_runs/{config_folder}/{run_id}/status.json` **last and unconditionally, including on + failure**; on `test` runs it also writes `validation.json` (count + title diff vs the manifest — + the WORKFLOW.md steps 22–25 QC report, produced indexer-side because only it holds AOSS + credentials). `status.json` carries `state ∈ succeeded|failed`, counts, `deletion_ratio`, + `deletion_mode` (deletes are **tombstones** — `public_visibility: false`, reversible), and a + machine-readable `error`: `export_not_found | export_incomplete | foreign_documents_in_scan | + scope_filter_ineffective | deletion_threshold_exceeded | deletion_budget_exceeded | + opensearch_error`. **Treat unknown `state` values as failure** — `needs_confirmation` is + reserved for future two-phase deletion. + +### What COSMOS must NOT do + +- **No `id`/`version` minting** — the indexer mints `id = /SDE/{config_folder}/|{url}` and a + content-hash `version` itself (`web/web_processor.py` is the sole owner of identity). COSMOS + exports raw curated fields only. +- **No AOSS, SageMaker, mapping, or deletion-guard concerns** — all live indexer-side. COSMOS gains + no ML or OpenSearch dependencies; only S3 writes, one `sts:AssumeRole`, and one `ecs:RunTask`. + +### Export traps (verified against the COSMOS models by the indexer team) + +- **`excluded` is a queryset annotation, not a field** — export with + `CuratedUrl.objects.filter(collection=c).exclude(excluded=True).iterator()`, or curator + exclusions get published. +- **`tdamm_tag` is a `PairedFieldDescriptor`** (manual over ML), not a column — `.values()` / + `.only()` on it fail; iterate instances. It is **exported but not indexed** (dropped by the + indexer's allow-list, excluded from its version hash, so a tag-only edit never re-vectorizes). +- `title = generated_title or scraped_title`, resolved at export time. +- `document_type` / `division` are **nullable ints** — resolve `DocumentTypes(v).label` / + `Divisions(v).label`, falling back to the collection's value. + +### Files (mirrors `sde-api-scrapers` W4.1–W4.7) + +| File | Change | +|---|---| +| `sde_collections/indexing/export.py` | **New.** `export_curated_to_s3(collection, target, run_id)` — stream `CuratedUrl`s per the traps above; write `documents.jsonl`, then `manifest.json` **last** | +| `sde_collections/indexing/dispatch.py` | **New.** `run_index_task(collection, target, run_id)` — assume the dispatch role, `ecs:RunTask` with the command override, return `taskArn` | +| `sde_collections/models/indexing.py` | **New** `IndexDispatch` (`collection` FK, `run_id`, `target`, `task_arn`, `dispatched_at`) — stall-timeout reference mirroring `ScrapeDispatch` (P3). Required because `CELERY_RESULT_BACKEND = None` means Celery task state cannot be polled | +| `sde_collections/tasks.py` | Fill the P5 stubs: `index_collection_to_test/prod` = mint `run_id` → export → dispatch → record `IndexDispatch` → set `TEST_INDEXING`/`PRODUCTION_INDEXING`. **New** `poll_index_runs()` | +| `sde_collections/signals.py` | `poll_index_runs` `PeriodicTask` every 2 min, gated on `INDEX_POLL_ENABLED` (same `post_migrate` pattern as P4's poller) | +| `sde_collections/utils/slack_utils.py` | Post `validation.json` to `sde-data-curation` (WORKFLOW.md step 24) | +| `config/settings/base.py` | `SDE_INDEX_BUCKET`, `INDEXING_ECS_CLUSTER`, `INDEXING_TASK_FAMILY`, `INDEXING_DISPATCH_ROLE_ARN`, `INDEX_POLL_ENABLED` — all **with defaults** so `config.settings.test` keeps booting | +| new migration | Creates `IndexDispatch` (additive) | + +`poll_index_runs` status mapping: `succeeded` + `test` → stay at `TEST_INDEXING`, post the +validation report to Slack, curator sets QC; `succeeded` + `prod` → `PROD_PERFECT` / `PROD_MINOR` +mirroring the QC status it entered with (WORKFLOW.md step 30); `failed`, unknown state, or stall +timeout (measured from `dispatched_at`) → `INDEXING_FAILED_ON_TEST` / `INDEXING_FAILED_ON_PROD`. +The `run_id` namespacing means an old run's `status.json` can never satisfy a newer dispatch — no +`LastModified` freshness rule needed (unlike the P4 crawler contract). + +### Cross-repo status (updated 2026-08-20 from their tracker's "Next Steps" — NS numbering is theirs) + +> **Correction: there is no cross-account boundary in dev.** Verified by the indexer team against +> AWS: the COSMOS Django host (`i-02b3d3e1ac0671952`, instance profile `indexing-helper-role`), the +> crawler, the indexer stacks, and the web AOSS collection all live in account **998871305517**. +> The earlier blocker "give them the COSMOS account id" had no unknown in it — the dev answer is +> the account both repos already deploy into. The assume-role indirection stays anyway (dispatch.py +> is unchanged), so test/prod can move to a genuinely separate account later. Test/prod account ids +> remain unknown but block nothing until those tiers exist (their NS.5). +> +> **Everything on both sides is code-complete, and the indexer is deployed to dev (NS.2 done +> 2026-08-20** — `web-indexing` merged to `develop`, PR #47 @ `c2aebad`, stacks deployed via CI). +> **NS.6 is also done (2026-08-20)** — the `sts:AssumeRole` grant is attached and was verified live +> from the COSMOS staging host. What remains is the network values in our env (NS.3) and the +> closed-loop proof (E2E.10). See `INDEXING_HANDOFF_TODO.md` for the step-by-step — note its C.* +> steps target the **staging** host `i-08f9b2175b70fa05c`, not production. + +- [x] **NS.1 (their side) — DONE 2026-08-13 (their W3.9):** `COSMOS_AWS_ACCOUNT_ID[DEV] = "998871305517"` + is filled, so their CDK now synthesizes the `sde-cosmos-indexing-dev` bucket policy and + `CosmosIndexingDispatchRole-dev`. Both grants name our host role + `arn:aws:iam::998871305517:role/indexing-helper-role` directly (`ArnPrincipal`, not + `AccountPrincipal`) — verified by rendering the CloudFormation. Consequence for us is NS.6. +- [x] **NS.2 (their side) — DONE 2026-08-20:** `web-indexing` merged to `develop` (PR #47 @ + `c2aebad`) and the stacks deployed to dev via CI. Stack outputs verified by the indexer team: + `sde-cosmos-indexing-dev` bucket, `CosmosIndexingDispatchRole-dev`, + `web_cosmos-scraper-dev:1` task def, `ScheduledRulesCount = 0` (correct — on-demand only). + The residual ECR check is also **done 2026-08-20**: `:latest` (digest `ae1c9e02…`) is + co-tagged with the exact merge commit `c2aebad…` and passed the offline tokenizer check + against that digest. Unblocks NS.3, NS.6, E2E.10. +- [ ] **NS.3 (their side → our env) — unblocked:** the Fargate network values (their cluster uses + the default VPC, so it's a lookup, not code; values were pre-looked-up 2026-08-18 — see + `INDEXING_HANDOFF_TODO.md` I.3 — and need re-confirming post-deploy). On receipt, set in + COSMOS dev env: `INDEXING_SUBNETS`, `INDEXING_SECURITY_GROUPS`, plus the known values + `SDE_INDEX_BUCKET=sde-cosmos-indexing-dev`, `INDEXING_ECS_CLUSTER=api-scrapers-cluster-dev`, + `INDEXING_TASK_FAMILY=web_cosmos-scraper-dev`, + `INDEXING_DISPATCH_ROLE_ARN=arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev`; + then `manage.py migrate` (the poller beat row is written at `post_migrate`) and restart. +- [x] **NS.4 (our side) — DONE 2026-08-13:** Phase 7 committed as `6fa19843` (+ plan `a3a1ab45`), + verified by the indexer team 2026-08-14, including `test_indexing_dispatch.py` and migration + `0080_indexdispatch`. +- [x] **NS.6 (our side) — DONE 2026-08-20:** inline policy `CosmosIndexingDispatch-dev` on + `indexing-helper-role` grants `sts:AssumeRole` on + `arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev` (a same-account trust is + satisfied only by the identity policy). Verified three ways: policy read back after + `put-role-policy`, `aws iam simulate-principal-policy` → `allowed`, and a live + `aws sts assume-role` **from the COSMOS staging host** returning credentials + (`Expiration = 2026-08-20T22:53:59+00:00`). This was the only IAM change needed on our side — + the indexer's bucket policy names our role directly, so no S3 statements were required. + P9's `preflight_aws` will *check* this grant; it did not create it. + **Host note:** the COSMOS indexing loop runs on the **staging** box + `i-08f9b2175b70fa05c` (`COSMOS Staging`, `18.215.146.207`, `ssh staging_cosmos`), not + production `i-02b3d3e1ac0671952`. Both share the `indexing-helper-role` instance profile, so + this grant covers either; NS.3's env values go on staging. `i-0178c998e868792d7` + (`COSMOS_Staging_Refresh`) has no instance profile and cannot dispatch. + Also attached `AmazonSSMManagedInstanceCore` to the role — neither COSMOS box had ever + registered with SSM; until an agent restart is confirmed, use SSH rather than + `ssm start-session`. +- [x] **NS.8 (our side) — DONE 2026-08-14 (`20c919ed`):** the `RunTask` command override was + flags-only; the indexer image has no `ENTRYPOINT`, and an ECS override replaces the command + wholesale, so every dispatched task would have died on "executable file not found". Fixed to + lead with `python3 api_scraper.py`; `test_indexing_dispatch.py` now pins the full command. + Container name (`WEB_COSMOSContainer`) and the omitted `--web-index` (so their + `WEB_INDEX_NAME` default governs — `sde-web-copy` at the time, `sde-web-subset` since 2026-08-19) were confirmed correct. +- [x] **Contract verified by the indexer team (2026-08-13)** — they read our `export.py` against + their `cosmos_source.py`/`web_processor.py`: manifest fields, JSONL names, label resolution, + `tdamm_tag`/`is_metadata_viewer` handling all match. *"Nothing to renegotiate."* +- [x] AOSS data-access policy for dev (their OOB.2) — **already satisfied**: the existing + `sde-services-access` policy's `index/sde-binary/sde-*` wildcard covers `sde-web-copy` **and** + `sde-web-subset`, with `ApiScraperTaskRole-dev` as principal. The subset mapping check + (**OOB.3**) was **verified 2026-08-20**: `sde-web-subset` carries the checked-in mapping + (`version: keyword`, knn 768/BINARY incl. nested full-text); census 59 docs across 2 + collections (`astromaterials_data_system` 52, `aurorasaurus_…` 7), none carrying `version` + (expected first-run profile), 0 id-scheme mismatches, 0 duplicate URLs. Their OOB.1 + (`version` on live `sde-web`) is **cutover-only**. +- [ ] **Cutover awareness:** flipping the indexer's `WEB_INDEX_NAME` from `sde-web-subset` (was `sde-web-copy` until 2026-08-19) to + `sde-web` *is* the production cutover; COSMOS needs no change for it. Their open + id-scheme-collision finding (12 collections, incl. 100% of `gcn_circulars`) is an + indexer-side guard/repair — COSMOS is unaffected but should not onboard those collections + until it lands. + +### Validation + +New `sde_collections/tests/test_indexing_dispatch.py` (AWS mocked): export writes the manifest +last; excluded URLs absent from the JSONL; tdamm/label resolution correct; `document_count` exact; +dispatch records an `IndexDispatch` row; the poller maps every `status.json` state (including +unknown-state-as-failure) and enforces the stall timeout; a stale run's `status.json` never +completes a newer dispatch. + +### Done when +- [x] Export/dispatch/poll implemented per the contract; `IndexDispatch` migration applied +- [x] P5 stubs replaced: a `CURATED` collection reaches `TEST_INDEXING`, and on `succeeded` the + validation report posts to Slack *(verified with mocked AWS; all settings default blank/off + so nothing can dispatch until the dev values are configured — dev is the only wired env)* +- [x] `QC_PERFECT`/`QC_MINOR` dispatches a prod run and lands on `PROD_PERFECT`/`PROD_MINOR` +- [x] Failure/stall paths land on `INDEXING_FAILED_ON_TEST`/`INDEXING_FAILED_ON_PROD` +- [ ] Closed loop verified against dev (their E2E.10): Curated → export → `RunTask` → poller → Slack + *(their NS.2 deploy DONE 2026-08-20; our NS.6 `sts:AssumeRole` grant DONE 2026-08-20 and + verified live from the staging host. The one remaining gate is NS.3: env values + + `migrate` + restart on `i-08f9b2175b70fa05c` (`ssh staging_cosmos`). Step-by-step in + `INDEXING_HANDOFF_TODO.md` C.2–C.4)* + +--- + +## Phase 8 — QC reporting: resolved, folded into Phase 7 + +**The validation script no longer needs to be authored — the indexer produces it.** On every +`--target test` run, the `WEB_COSMOS` task writes `validation.json` +(`expected_count`, `indexed_count`, `count_matches`, `titles_missing_in_index`, +`titles_only_in_index`, `title_match_rate` — exactly the WORKFLOW.md steps 22–25 count/title +comparison) next to `status.json`. It lives indexer-side by design: the indexer already holds AOSS +credentials and COSMOS has none. Its counts exclude tombstoned documents. + +COSMOS's entire share of this phase is inside P7: `poll_index_runs` reads `validation.json` and +posts it to `sde-data-curation`. The QC statuses stay **curator-set from the report**, so nothing +else is needed on the status side after Phase 1. + +### Done when +- [x] Covered by P7's done-when (validation report posted to Slack on test runs) — no separate work + +--- + +## Phase 9 — CI/CD: deploy, rollback, preflight + +**Goal:** implement `sde_collections/DEPLOYMENT.md`. Independent of Phases 3–8 — **authoring can +run in parallel once Phase 1 lands**, but the first deploy to a *fresh* host requires Phase 6: +the Sinequa settings are required env vars with no defaults (see P6), so until they're deleted a +clean host needs dummy Sinequa secrets to boot. + +> **Note:** the earlier draft of this machinery is **not recoverable**. This clone has a 2-entry +> reflog, no stashes, and zero hits for `preflight_aws` / `validate_deploy_env` / `ecr.override` / +> `OPENSEARCH_ENDPOINT` across every commit in every ref. The only prior art is commit +> `08c3ef30` on the unmerged branch `94-add-cicd-…`, a 51-line `deploy.yml` stub with a placeholder +> role ARN and everything after the AWS-credentials step commented out. Write from scratch. + +### Files + +| File | Content | +|---|---| +| `sde_collections/management/commands/validate_deploy_env.py` | **New.** Fails if required settings are missing. Once P7 lands, it must also require the indexing settings (`SDE_INDEX_BUCKET`, `INDEXING_ECS_CLUSTER`, `INDEXING_TASK_FAMILY`, `INDEXING_DISPATCH_ROLE_ARN`, and the Fargate networking pair `INDEXING_SUBNETS`/`INDEXING_SECURITY_GROUPS`) non-empty on deployed hosts. (Test/prod endpoint separation is enforced **indexer-side** — its target→endpoint resolution is tier-capped — so COSMOS has no endpoint-equality check to make) | +| `sde_collections/management/commands/preflight_aws.py` | **New.** SSM reachability to the crawler instance and S3 read on `SDE_S3_BUCKET`, using `get_boto3_session()` from P0. Reports each check independently rather than aborting on the first failure. P7 adds checks on `SDE_INDEX_BUCKET` (write `curated_collections/*`, read `index_runs/*`) and an `sts:AssumeRole` on the dispatch role (the grant itself is NS.6, made manually once the role is deployed — preflight verifies it, it does not create it); COSMOS never gets AOSS/SageMaker access, so no such checks exist here | +| `config/urls.py` + a `healthz` view | **New.** There is **no health endpoint in the repo today**; the deploy smoke checks need one. Minimal 200 + DB connectivity, unauthenticated | +| `.pre-commit-config.yaml` | **Fix:** the gitleaks hook passes `--config=gitleaks-config.toml`, but that file does not exist and is not tracked — the hook fails instead of scanning. Add the config or drop the arg | +| `scripts/deploy.sh` | **New.** The single definition of a deploy, in `DEPLOYMENT.md`'s order: fetch artifact → `validate_deploy_env` → backup (prod only) → `migrate` → `docker compose up -d` → smoke checks → rollback on failure | +| `ecr.override.yml` | **New.** Compose override pulling the Django image from ECR | +| `.github/workflows/ci.yml` | **New.** `run-tests` and `django-checks` (`check --deploy`, `makemigrations --check --dry-run`) on PRs into `dev`, `staging`, **and** `production` — today only `dev` is covered, so the release path itself is untested. Drop `init.sh`'s per-file loop (a process per test file, re-paying Django setup each time) in favour of a single `pytest` run | +| `.github/workflows/deploy-staging.yml` | build → ECR → SSM → `deploy.sh --environment staging` | +| `.github/workflows/deploy-production.yml` | staging-digest check → SSM → `deploy.sh --environment production`, gated on the `production` GitHub Environment | +| `.github/workflows/rollback.yml` | Manual redeploy of a named image tag | +| `.github/workflows/secret-scan-history.yml` | Weekly full-history gitleaks, report-only | +| `.github/workflows/run_full_test_suite.yml` | **Delete** — superseded by `ci.yml` | + +All `deploy-*` and `rollback` workflows gate on repo variable **`CD_ENABLED`**; until it is `true` +they skip. + +> **Ordering matters:** `validate_deploy_env` runs *before* anything mutates. A host missing +> `INDEXING_DISPATCH_ROLE_ARN` must fail there, not inside a Celery task that has already exported +> a collection it can never dispatch. + +> **`celerybeat` must be recreated on every deploy.** The inference `PeriodicTask` rows (P2) and the +> `poll_scrape_jobs` schedule (P4) are written by `post_migrate`; a beat process left running on the +> old schedule silently ignores them. + +### Validation + +```bash +docker-compose -f local.yml run --rm django python manage.py validate_deploy_env # expect clear failure locally +docker-compose -f local.yml run --rm django python manage.py preflight_aws # against sde-dev +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_deploy_commands.py +bash -n scripts/deploy.sh && shellcheck scripts/deploy.sh +``` + +Tests (AWS mocked): `validate_deploy_env` exits non-zero on any missing required setting; +`preflight_aws` reports each check independently and does not abort on the first failure. + +### Manual verification +1. `aws ssm describe-instance-information --filters "Key=InstanceIds,Values=" --query 'InstanceInformationList[0].PingStatus'` → `Online` +2. `aws ecr describe-repositories --repository-names cosmos` → no error +3. Deploy to **staging** only, then **rehearse a rollback on staging** before trusting production. + +### Done when +- [ ] Both management commands exist with tests +- [ ] `scripts/deploy.sh` passes `shellcheck`; runs end to end on staging +- [ ] Five workflows added, `run_full_test_suite.yml` removed, CI runs on all three branches +- [ ] `/healthz` endpoint added and asserted by the deploy smoke checks +- [ ] gitleaks hook actually runs (missing `gitleaks-config.toml` resolved) +- [ ] **Production DB credential rotated** — it is live in committed `HEAD` at `SQLDumpRestoration.md:101,117` along with the RDS hostname; scrub the file after rotating +- [ ] A rollback rehearsed successfully on staging +- [ ] `CD_ENABLED` prerequisites documented as met (or explicitly pending) + +--- + +## End-to-end verification (after Phases 0–6) + +```bash +docker-compose -f local.yml build && docker-compose -f local.yml up -d +docker-compose -f local.yml run --rm django python manage.py migrate +docker-compose -f local.yml run --rm django pytest +``` + +Then, against the dev AWS account: + +1. Create a collection in the admin with a real `seed_url` and a `division`. +2. Set **Research in Progress → Ready for Engineering**. +3. Confirm the job JSON reaches `/opt/sde-crawler/jobs/incoming/.json` and moves to + `jobs/done/`. +4. Confirm `s3:///scraped_collections/.json` and the + `_failures_summary.json` both appear. +5. Watch `docker-compose -f local.yml logs -f celeryworker` — the poller ingests, status becomes + **Scraping Successful**, then **Ready for Curation**. +6. `shell_plus`: `c.dump_urls.count()` matches `documents_scraped`; `c.delta_urls.count() > 0`. +7. Curate, set **Curated**, confirm `CuratedUrl`s exist and `DeltaUrl`s are cleared. +8. Confirm Slack received the ingest summary and the curation message. +9. **Re-scrape:** set `reindexing_status` to **Re-Indexing Needed** on the same collection — + confirm a *new* `ScrapeDispatch` row, that the poller ignores the old S3 output until fresh + results land, and that the collection reaches **Ready for Re-Curation**. + +Steps beyond 8 (test indexing, validation, prod indexing) need **Phase 7**. The indexer side is +code-complete **and deployed to dev (NS.2 done 2026-08-20** — merged to `develop`, PR #47; account +id recorded and AOSS data-access policy already in place), and our `sts:AssumeRole` grant is in +place too (**NS.6 done 2026-08-20**, verified live from the staging host). The closed loop now needs +one thing: the Fargate network values in our env + `migrate` + restart on the **staging** host +`i-08f9b2175b70fa05c` (`ssh staging_cosmos`) — NS.3. Until then status stops at `Test Indexing`. +Step-by-step: `INDEXING_HANDOFF_TODO.md`. + +--- + +## Cross-cutting risks + +| Risk | Mitigation | +|---|---| +| Colour-map `KeyError` breaks the collection list/detail pages | P1 converts both lookups to `.get(..., "btn-light")` and adds a parametrised test over every enum member | +| Three TDAMM collections stall forever when inference is disabled | P2 short-circuits `queue_necessary_classifications()` itself, not just the beat schedule; regression test included | +| Credential-model mismatch (static keys in code vs instance role in `DEPLOYMENT.md`) | P0's `get_boto3_session()` is the single entry point; no new static-key call sites. Helper reads `SDE_AWS_*`, never the django-storages `AWS_ACCESS_KEY_ID` (absent from `base.py` → `AttributeError` under test, and wrong credential scope) | +| Zero-page crawl silently produces an empty collection | P4 treats `documents_scraped == 0` as `SCRAPING_FAILED` | +| **Stale S3 results ingested after a re-dispatch** | P3's `ScrapeDispatch.dispatched_at` + P4's freshness rule: results with `LastModified` before the latest dispatch are invisible to the poller | +| **Re-scrape (reindexing) flow severed when `fetch_full_text` is deleted** | P3 dispatches on `REINDEXING_NEEDED_ON_DEV`; P4 polls it and flips `REINDEXING_FINISHED_ON_DEV` itself; P5 removes the old trigger branch; P6's deletion is then safe | +| Duplicate ingest from the 5-minute poller | Ingest claims via an atomic status CAS **before** writing (ingest can outrun the poll interval, and `BaseUrl.url` is globally unique — concurrent writes would `IntegrityError`); manual re-ingest stays idempotent by deleting `DumpUrl`s first | +| Deleting Sinequa breaks unrelated GitHub metadata sync | P6 keeps `sync_with_production_webapp` (COSMOS prod webapp) and removes only the Sinequa-configs-repo paths | +| `config/settings/test.py` inherits `base.py`, so stale env vars mask missed references | P6 validation runs with the deleted vars unset | +| ~~**Cross-account dispatch silently unbuildable**~~ — `sde-api-scrapers` had no COSMOS account id, so its bucket policy and `CosmosIndexingDispatchRole` were not synthesized | **Resolved 2026-08-13**: dev is the same account (`998871305517`), filled in their `settings.COSMOS_AWS_ACCOUNT_ID` (their W3.9); both resources now synthesize and name `indexing-helper-role` directly. **Deployed 2026-08-20 (NS.2)**; our `sts:AssumeRole` grant landed the same day (NS.6, verified from the staging host) — remaining: env values on staging (NS.3) | +| **`RunTask` command override must lead with the executable** — the indexer image has no `ENTRYPOINT`, and an ECS override replaces the whole command | Fixed in `20c919ed` (NS.8); `test_indexing_dispatch.py` asserts the full `python3 api_scraper.py …` command so a regression fails a test | +| Curator-excluded URLs published to search | P7 export uses `.exclude(excluded=True)` — `excluded` is a queryset annotation, not a field; test asserts exclusions never reach the JSONL | +| Truncated export read as a mass deletion | Indexer-side guard: line count vs `manifest.document_count` → `export_incomplete`, deletions skipped. COSMOS's job is writing the manifest **last** with an exact count | +| Stale `status.json` completes a newer index dispatch | Every artifact is namespaced by the COSMOS-minted `run_id` (`index_runs/{cf}/{run_id}/`), so cross-run staleness is structurally impossible — unlike the P4 crawler contract, which needs the `LastModified` freshness rule | +| `factories.py:49` has a suspect `tracker = factory.Maybe("workflow_status")` | Watch during P1/P5 test rewrites; fix if it interferes with `FieldTracker` | +| Migration rollback safety | All migrations here are additive (new model, choices-only `AlterField`); `DEPLOYMENT.md`'s "image rollback is a complete rollback" property holds | diff --git a/INDEXING_HANDOFF_TODO.md b/INDEXING_HANDOFF_TODO.md new file mode 100644 index 00000000..576e4a7d --- /dev/null +++ b/INDEXING_HANDOFF_TODO.md @@ -0,0 +1,755 @@ +# Web Indexing Hand-off — Status Update & Remaining To-Do (Indexer + COSMOS) + +## Status overview — 2026-08-20 + +> **Verified 2026-08-18** against `COSMOS@9e18ced8` (working tree), `sde-api-scrapers@10975ee`, and the live +> `sde-dev` account (`998871305517`). Corrections made in this pass: `get_chunker` import path (I.1), C.2 needs +> `manage.py migrate` for the beat row, C.3 S3 probes rewritten (role has `AmazonS3FullAccess`; bucket policy has +> no `ListBucket`), `dispatch_scrape --collection`, ratio guard is `>` not `≥`, four (not three) ownership +> boundaries, `+67` tests (not `47+`), I.3 subnet/SG values pre-filled, `:latest` overwrite caveat in I.1, and +> the uploader endpoint tier-cap gap recorded as an indexer to-do. +> +> **Retarget 2026-08-19 (`10975ee`):** the branch-era working index is now **`sde-web-subset`**, a scratch +> *subset* of live `sde-web`, replacing `sde-web-copy` (the full 431k-doc copy) as every runtime default +> (`web/web_pipeline.py`, `infrastructure/config/settings.py`, `api_scraper.py`, `scripts/audit_index.py`) and +> the value pinned by `tests/test_infra_web_env.py`. `sde-web-copy` still exists in dev AOSS but nothing on the +> branch targets it. Verifications dated before 2026-08-19 in the indexer tracker were done against the copy; +> the tracker's new **OOB.3** (confirm `sde-web-subset` carries the `version: keyword` mapping + access-policy +> coverage) must pass before I.4 *(it did — verified 2026-08-20, see below)*. This file has been +> updated to the subset throughout. +> +> **Deployed 2026-08-20 (per the indexer tracker):** `web-indexing` was **merged to `develop` (PR #47 @ +> `c2aebad`) and the stacks deployed to dev — NS.2 (I.1/I.2 here) is DONE.** Stack outputs verified by the +> indexer team: `sde-cosmos-indexing-dev` bucket, `CosmosIndexingDispatchRole-dev` +> (`arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev`), task def `web_cosmos-scraper-dev:1`, +> and `ScheduledRulesCount = 0` (correct — `WEB_COSMOS` is on-demand only). The merge also closes **NS.7** +> (the `deploy.yml` test job now guards this code) and moots I.1's hand-push instructions. +> +> **Both pre-run sanity checks passed 2026-08-20 (per the indexer tracker):** +> - **ECR image check DONE** — `:latest` (digest `ae1c9e02…`) is co-tagged `c2aebadcea62…`, the exact +> PR #47 merge commit, pushed by CI at 14:54 CDT. The E2E.1 offline tokenizer check was re-run +> against that exact digest and passed (`HF_HUB_OFFLINE=1`, `web/` present, `get_chunker()` with +> zero network). Note: the image is **linux/amd64 only** — correct for Fargate; local runs on +> Apple Silicon need `--platform linux/amd64`. +> - **OOB.3 VERIFIED** — `sde-web-subset` exists in dev AOSS with the checked-in mapping (`version: +> keyword`; `vectorized_title` knn 768/BINARY; nested `vectorized_full_text` likewise; +> `public_visibility: boolean`). Census: **59 docs across 2 collections** +> (`astromaterials_data_system` 52, `aurorasaurus_reporting_auroras_from_the_ground_up` 7); +> **0 of 59 carry `version`**, so the first run of either collection has the expected first-run +> profile (empty state scan, full vectorize, zero deletions — see `first_run.md` / +> `COSMOS_INDEX_REAL_RUN.md` §4); 0 id-scheme mismatches, 0 duplicate URLs. + +> **Host correction 2026-08-20 — COSMOS work happens on STAGING, not production.** Every C.* step +> below now targets **`i-08f9b2175b70fa05c`** (tag `COSMOS Staging`, `18.215.146.207`, ssh alias +> `staging_cosmos`, user `ec2-user`, key `~/.ssh/sde-indexing-helper-staging.pem`). Earlier revisions +> of this file named `i-02b3d3e1ac0671952` (tag `COSMOS`, `54.227.74.92`, ssh alias +> `production_cosmos`) as "the COSMOS host" throughout — that is the production box and is **not** +> where this loop runs. **Nothing was executed on it.** This retarget costs no IAM rework: staging +> and production carry the *same* instance profile, `indexing-helper-role`, so C.1's grant (which +> lives on the role, not the host) applies to staging unchanged, as does the indexer dispatch role's +> trust policy. Watch out for the third box, **`i-0178c998e868792d7`** (tag `COSMOS_Staging_Refresh`, +> `23.20.135.115`) — it has **no instance profile at all**, so no dispatch can ever work from it; +> C.2 must land on `i-08f9b2175b70fa05c` specifically. +> +> **C.1 / NS.6 DONE 2026-08-20 — verified end to end.** Inline policy `CosmosIndexingDispatch-dev` +> is attached to `indexing-helper-role` (verified by read-back), `aws iam simulate-principal-policy` +> returns **`allowed`** for `sts:AssumeRole` → `CosmosIndexingDispatchRole-dev`, the dispatch role's +> trust policy names `arn:aws:iam::998871305517:role/indexing-helper-role`, and **`aws sts +> assume-role` run on the staging host itself succeeded** (`Credentials.Expiration = +> 2026-08-20T22:53:59+00:00`). Dispatch IAM is proven from the box that will do the dispatching. +> **Next up: C.2.** +> +> **SSM is not usable on these hosts yet.** `indexing-helper-role` carried no SSM permissions at all, +> so neither COSMOS box has ever registered as a managed instance (`describe-instance-information` +> returns nothing). `AmazonSSMManagedInstanceCore` was attached to the role 2026-08-20 — which covers +> staging too, same role — but the production box had not registered after ~15 min of polling, so the +> agent may also need a restart (`sudo systemctl restart amazon-ssm-agent`) or may not be installed. +> **Until SSM works, use SSH** (`ssh staging_cosmos`) for C.1's verification and for C.2/C.3. + +**One sentence:** the indexer is **deployed to dev and pre-run-verified** (image + subset both checked +2026-08-20); what stands between us and a working Curated → indexed loop is confirming one IAM grant +on-host (C.1 — the policy is already attached), env values on the COSMOS **staging** host (C.2–C.3), +and the end-to-end proofs (I.4 / C.4) — **I.4 is runnable right now with no prerequisites left**. + +## What's next — step by step + +The exact commands for each step live in the detail sections referenced in parentheses. +Steps 1–3 are COSMOS wiring; 4 is indexer-side and **fully unblocked — it can start immediately +and run in parallel with 1–3**; 5 is the closed loop. + +1. ~~**C.1 — grant `sts:AssumeRole`** on `indexing-helper-role` for + `CosmosIndexingDispatchRole-dev` — one `put-role-policy`~~ — **DONE 2026-08-20**: policy + attached, simulator `allowed`, and `aws sts assume-role` verified on the staging host itself. +2. **C.2 — wire the staging host** — append the env vars (bucket, cluster, family, role ARN, + subnets/SGs — the I.3 values from 2026-08-18 are pre-filled; re-run the lookup once to confirm + they're unchanged post-deploy — and `INDEX_POLL_ENABLED=true`), then **`manage.py migrate`** + (required — the `poll_index_runs` beat row is written by `post_migrate`; a restart alone does + nothing), then restart django/celeryworker/celerybeat. +3. **C.3 — pre-flight** — S3 smoke probes + a dry `run_index_task` dispatch from a Django shell + against a scratch collection. +4. **I.4 — CLI end-to-end against `sde-web-subset`** *(E2E.2–9c)* — hand-written 10-doc export → + manual `run-task` → `status.json` / `validation.json` / index checks, deletion-guard and + id-collision-guard cases. **No prereqs remain** (image check + OOB.3 both passed 2026-08-20); + does not wait on COSMOS. Read `first_run.md` before the first run. +5. **C.4 — closed loop** *(E2E.10, the last open Phase 7 done-when)* — set a small non-collision + collection to **Curated** → export → `RunTask` → poller → Slack report → QC → prod dispatch → + `PROD_PERFECT`; plus the failure path. Needs steps 1–3. Note the subset already holds two + collections (`astromaterials_data_system`, `aurorasaurus_…`) with clean ids and no `version` — + per the census, a run against either is a plain first run (no collision-guard refusal), so + whichever curated collection is chosen behaves predictably. +6. **Close out** — tick E2E boxes in the indexer tracker, "Closed loop verified" + NS.3 in + `IMPLEMENTATION_PLAN.md`; then Phase 9 CI/CD (independent — don't let it gate the loop) and the + not-blocking housekeeping (C.5 / I.5: uploader tier-cap gap before any prod deploy, id-collision + *repair*, NS.5, OOB.1 at cutover only). + +**Headline numbers** + +| | Indexer (`sde-api-scrapers` · merged to `develop`, PR #47 @ `c2aebad`) | COSMOS (`COSMOS` · `cosmos-rewiring` · HEAD `9e18ced8`) | +|---|---|---| +| Phases | W0–W5 all done (6 of 6); **stacks deployed to dev 2026-08-20 (NS.2)** | P0–P6 done, **P7 10 of 14 boxes** (4 open, see below), P8 resolved (folded into P7), **P9 not started, 0 of 8** | +| Tests | **316** offline, all green — re-run 2026-08-18 (`uv run pytest tests/ -q`, 316 passed); `test_fixes.md` mutation audit: 17/17 killed; CI now runs the suite on `develop` pushes | `test_indexing_dispatch.py` + rewritten workflow tests, all green (run via `docker compose -f local.yml run --rm django pytest`; not re-run 2026-08-18 — no local Django env) | +| In AWS today (per indexer tracker, 2026-08-20, `sde-dev`) | **Deployed and pre-run-verified**: `sde-cosmos-indexing-dev` bucket, `CosmosIndexingDispatchRole-dev`, `web_cosmos-scraper-dev:1` task def, 0 scheduled rules (correct); ECR `:latest` = merge commit `c2aebad` (digest `ae1c9e02…`, tokenizer check passed); `sde-web-subset` mapping verified, 59 docs / 2 collections, none versioned (OOB.3) | Django **staging** host `i-08f9b2175b70fa05c` (`COSMOS Staging`, `18.215.146.207`), instance profile `indexing-helper-role` — now carrying inline `CosmosIndexingDispatch-dev` (C.1, 2026-08-20) plus `AmazonS3FullAccess`, `AmazonRDSFullAccess`, `indexing-helper-s3-access`, `AmazonSSMManagedInstanceCore`; every dispatch-gating setting blank/off by design | +| Blocking dependency | **none — E2E.2–9c runnable now** | **unblocked** — C.1 applied (on-host confirm outstanding); C.2/C.3 can run now; C.4 needs them done | + +### Indexer — done + +- **Pipeline** (`WEB_COSMOS` source): S3 export → chunk → SageMaker vectorize → AOSS bulk upsert into the shared web index, with the safety stack the shared index demands — scope probe, filtered state scan + ownership assertion at four boundaries (`state_scan`, `export_ids`, `upsert_batch`, `deletion_candidates`), export-completeness check, deletion ratio (abort when > 0.90) + absolute cap (5,000, checked first), reversible tombstones, `status.json` written last and always, `validation.json` on test runs. +- **New since last update (`cf695f9`, 2026-08-18):** W2.13 **id-collision guard** — refuses a run when the index already holds the collection under ids the pipeline would not mint (`id_scheme_collision`) or under duplicated ids (`duplicate_business_ids`), before any spend; `--allow-id-collision` is an audited override. This closes the "12 collections would be silently doubled" finding. Plus `first_run.md` (what run 1 actually does) and a test-suite hardening pass (+67 tests, incl. new `test_tombstone_batch.py`, `test_ensure_index.py`, `test_id_collision_guard.py`). +- **Infrastructure (CDK)**: `sde-cosmos-indexing-{env}` bucket with per-prefix/per-direction policy, `CosmosIndexingDispatchRole-{env}` (trusts COSMOS's `indexing-helper-role` only, `RunTask` limited to the `web_cosmos-scraper-{env}` family + cluster), on-demand task def with all env injected and pinned by tests, monitoring filter so an on-demand task doesn't park an alarm, `AWSV4SignerAuth` fix for long runs. `cdk synth -c environment=dev` re-verified clean 2026-08-18 (test/prod per the tracker). `COSMOS_AWS_ACCOUNT_ID[DEV]` filled — dev is **same-account** (`998871305517`). +- **Contract with COSMOS** verified field-by-field on 2026-08-13 — nothing to renegotiate. Every default targets **`sde-web-subset`**, a scratch subset of live `sde-web` (replaced `sde-web-copy` 2026-08-19); production is reachable only by an explicit override at cutover. +- **CI**: `deploy.yml` `test` job gates the image build (the tracker's earlier "no CI" note was wrong and has been retracted). Triggers: pushes to `develop`/`test`/`main` **and `workflow_dispatch`** — a manual run on `web-indexing` is possible. +- **Known latent gap (not a dev blocker):** only the pipeline's probe/validate client is tier-capped by `--target` (`OPENSEARCH_ENDPOINT_TEST/PROD`); `APIOpenSearchUploader` (`fetch_index_state`, `index_batch`, `tombstone_batch`) always uses bare `OPENSEARCH_ENDPOINT`. Identical in dev (all three resolve to the same collection), but on a prod deploy `--target test` would probe test while writing prod. Track as an indexer to-do before cutover. + +### Indexer — remaining + +1. ~~Build/push the image and deploy (I.1–I.2, NS.2)~~ — **DONE 2026-08-20** via merge to `develop` + (PR #47) + stack deploy. ~~Residual ECR image check~~ — **DONE 2026-08-20**: `:latest` co-tagged + with the merge SHA; tokenizer check re-run against that exact digest. +2. ~~OOB.3~~ — **VERIFIED 2026-08-20**: `sde-web-subset` carries the checked-in mapping; census + recorded (59 docs, 2 collections, none versioned). +3. **Confirm the Fargate subnet/SG ids unchanged** (I.3, NS.3) — values already looked up 2026-08-18; + re-confirm against the deployed cluster and paste into C.2. +4. **CLI end-to-end against `sde-web-subset`** (I.4, E2E.2–9c: E2E.1 done — re-verified against the + deployed image; 14 boxes open) — **no prereqs remain**; does not wait on COSMOS. +5. Not blocking: test/prod account ids (NS.5); `version: keyword` on live `sde-web` (OOB.1, cutover only); + the id-scheme *repair* (guard is in, repair isn't); the uploader endpoint tier-cap gap above (before any + prod deploy). + +### COSMOS — done + +- **P0–P2** settings/session helper, 21–26 workflow statuses + colour maps + Slack transitions, inference pipeline disabled (not deleted). +- **P3–P4** scrape dispatch to the crawl4ai host via SSM (`ScrapeDispatch` freshness/stall model) and S3 → `DumpUrl` ingest with atomic claim + 5-min poller. +- **P5** curation triggers: `CURATED` → test-index hand-off, `QC_PERFECT/QC_MINOR` → prod hand-off, re-scrape path. +- **P6** Sinequa deleted. +- **P7** indexing hand-off — export (manifest last, exact count, exclusions honoured), `sts:AssumeRole` → `ecs:RunTask` with the correct command override (a flags-only bug was found and fixed 2026-08-14), `IndexDispatch` model + migration, 2-min S3 poller with unknown-state-as-failure and 6-h stall, Slack validation report; every dispatch-gating setting (bucket, cluster, family, role ARN, subnets, SGs, both `*_POLL_ENABLED`) defaults blank/off so nothing dispatches until wired (`INDEXING_CONTAINER_NAME` defaults to `WEB_COSMOSContainer`, `INDEX_STALL_TIMEOUT_HOURS` to 6, `AWS_REGION` to `us-east-1`). Note: `run_index_task` fail-fasts only on role ARN/cluster/family and `export_curated_to_s3` on the bucket; blank subnets/SGs surface as an AWS-side `RunTask` error, and an empty curated set (`document_count == 0`) fails dispatch → `INDEXING_FAILED_ON_TEST`. **11 of 14 P7 boxes done (in the working-tree `IMPLEMENTATION_PLAN.md` — uncommitted; committed HEAD still shows 6 of 12). NS.6 ticked 2026-08-20. Open: NS.3, closed loop (both unblocked) + "Cutover awareness" (needs no COSMOS change).** +- **P8** resolved — the QC validation report is produced indexer-side. + +### COSMOS — remaining + +1. ~~**`sts:AssumeRole` grant** on `indexing-helper-role` for `CosmosIndexingDispatchRole-dev` (C.1, NS.6)~~ — **DONE 2026-08-20**: inline `CosmosIndexingDispatch-dev` attached, simulator `allowed`, on-host `assume-role` from staging returned credentials. **NS.6 can be ticked in `IMPLEMENTATION_PLAN.md`.** +2. **Set the env vars on the STAGING host (`i-08f9b2175b70fa05c` / `ssh staging_cosmos`), `migrate`, and restart** (C.2) — bucket, cluster, task family, dispatch role ARN, subnets, SGs, `INDEX_POLL_ENABLED=true` (container name already defaults). The `poll_index_runs` beat row is created/enabled by a `post_migrate` receiver, so `manage.py migrate` is required — a restart alone does not enable it. +3. **Pre-flight + closed loop** (C.3–C.4, their E2E.10): Curated → export → RunTask → poller → Slack → QC → prod. This is the last open Phase 7 done-when. +4. **Phase 9 CI/CD** (deploy/rollback/preflight, healthz, gitleaks, **prod DB credential rotation** — 0 of 8 done) — independent of the indexing loop; don't let the loop wait on it. +5. Keep the 12 id-collision collections out of the flow until the indexer's *repair* lands (the guard will refuse them fast if one slips through). Cutover needs no COSMOS change. + +### Order of operations + +`C.1 → C.2 → C.3 → C.4`, with `I.4` runnable **immediately, in parallel** — I.1/I.2, the ECR image +check, and OOB.3 are all done (2026-08-20), so every remaining gate is COSMOS-side wiring or a proof. + +--- + +## Detail + +**Status 2026-08-20** (indexer merged to `develop` — PR #47 @ `c2aebad`, stacks deployed to dev; COSMOS HEAD `9e18ced8`): all code on both sides is complete and +committed (COSMOS's `IMPLEMENTATION_PLAN.md` Phase 7 box updates and this file are still uncommitted) — the indexer suite is **316** offline tests, incl. the **W2.13 id-collision guard** +(`web/id_collision.py`). **The deploy (I.1/I.2, NS.2), the ECR image check, and OOB.3 are all done +(2026-08-20)**; what remains is COSMOS-side wiring (C.1–C.3) and the end-to-end proofs (I.4, C.4). +I.1/I.2 below are kept as a record and as the fallback procedure; skip to I.3. + +- Indexer repo: `sde-api-scrapers`, now on `develop` (tracker: `Web Indexing - Task Plan & Tracking.md`; first-run walkthrough: `first_run.md`; test audit: `test_fixes.md`) +- COSMOS repo: `COSMOS`, branch `cosmos-rewiring` (tracker: `IMPLEMENTATION_PLAN.md`, Phase 7) +- Account (dev, both sides): **998871305517**, profile `sde-dev`, region `us-east-1` +- Working index for every branch-era run: **`sde-web-subset`** — a scratch subset of live `sde-web`, since 2026-08-19 (`sde-web-copy` before that; never live `sde-web` until cutover) + +Order of operations: **~~I.1 → I.2~~ (done) → I.3 → C.1 → C.2 → C.3 → I.4/C.4** (closed loop); I.4 has no open prereqs (image check + OOB.3 both done 2026-08-20). + +--- + +## Indexer (`sde-api-scrapers`) + +### I.1 — Build and push the `web-indexing` image to ECR (`api-scrapers-dev:latest`) — **DONE 2026-08-20 via merge** + +> **DONE 2026-08-20:** the preferred path below was taken — `web-indexing` merged to `develop` +> (**PR #47 @ `c2aebad`**), so CI's `test → build+push → cdk deploy` covered I.1 and I.2 in one run. +> **Residual image check also DONE 2026-08-20:** `:latest` (digest `ae1c9e02…`) is co-tagged +> `c2aebadcea62…` — the exact merge commit — pushed by CI at 14:54 CDT; the offline tokenizer check +> was re-run against that exact digest and passed. The image is **linux/amd64 only** (GitHub runner +> build) — correct for Fargate; local runs on Apple Silicon need `--platform linux/amd64`. The +> hand-push commands stay as the fallback. + +The ECS task definitions pull `{ecr}/api-scrapers-dev:latest`. CI (`deploy.yml`) only builds on +pushes to `develop`/`test`/`main` (or a manual `workflow_dispatch`), so a branch deploy must push the image by hand — +**or merge to `develop` and let CI do I.1 + I.2 in one run** (see below). + +> **Checked 2026-08-19:** `deploy.yml` does `test → build+push (:sha, :latest) → cdk deploy --all → verify`, so a +> CI run covers I.1 *and* I.2. But `GitHubActions-ApiScrapers-DEV`'s trust policy is pinned to +> `repo:NASA-IMPACT/sde-api-scrapers:ref:refs/heads/develop` (PROD to `refs/heads/main`), so a +> `workflow_dispatch` on `web-indexing` fails at "Configure AWS credentials". **Preferred path: merge +> `web-indexing` → `develop` and push** (`git checkout develop && git merge --no-ff web-indexing && git push`, +> then `gh run watch`); rollback is `git revert -m 1 ` + push. The hand-push below remains the +> alternative if you'd rather not merge before I.4 (it would need the trust policy widened to reach CI). + +> **Caveat (verified 2026-08-18):** `api-scrapers-dev:latest` is currently the `develop` build (`7b4c8b0`, +> pushed 2026-08-14) and is what the 8 existing scheduled dev scrapers (`cmr_api`, `gcn_circulars`, …) +> pull on every run. Pushing `web-indexing` as `:latest` puts branch code under those tasks too — acceptable +> in dev only if the branch is a superset of `develop` (it is, per the tracker) and you're prepared to re-push +> `develop` if anything regresses. The Dockerfile +bakes the HF tokenizer at build time (W0.2) — build needs network access. + +```bash +cd ~/projects/sde-api-scrapers +git checkout web-indexing && git pull +export AWS_PROFILE=sde-dev AWS_REGION=us-east-1 +ACCOUNT=998871305517 +ECR=$ACCOUNT.dkr.ecr.us-east-1.amazonaws.com/api-scrapers-dev + +aws ecr get-login-password | docker login --username AWS --password-stdin $ACCOUNT.dkr.ecr.us-east-1.amazonaws.com +docker build --platform linux/amd64 -t $ECR:latest -t $ECR:$(git rev-parse --short HEAD) . +docker push $ECR:latest +docker push $ECR:$(git rev-parse --short HEAD) + +# sanity: tokenizer resolves offline inside the image (E2E.1) +docker run --rm --network none $ECR:latest python3 -c "from uploader.text_chunker import get_chunker; get_chunker(); print('ok')" +``` + +- [x] Confirmed 2026-08-20: `:latest` digest `ae1c9e02…` co-tagged with merge SHA `c2aebad…`, CI-pushed 14:54 CDT; offline tokenizer check passed against that digest (E2E.1 re-verified). + +### I.2 — Deploy the branch stacks to dev (**NS.2**) — **DONE 2026-08-20** + +> **DONE 2026-08-20** by the merge's CI run. Stack outputs verified (indexer tracker NS.2): +> `CosmosIndexBucketName = sde-cosmos-indexing-dev`, `CosmosIndexingDispatchRoleArn = +> arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev`, `WEBCOSMOSTaskDefArn = +> …:task-definition/web_cosmos-scraper-dev:1`, `ScheduledRulesCount = 0` (correct — on-demand only). +> The verification commands below remain useful spot checks and the checkboxes are ticked +> accordingly; re-run them if anything looks off. + +Creates the `sde-cosmos-indexing-dev` bucket (+ bucket policy naming `indexing-helper-role`), +`CosmosIndexingDispatchRole-dev`, the `web_cosmos-scraper-dev` task family, and the monitoring +filter. Requires `COSMOS_AWS_ACCOUNT_ID[DEV]` to be filled (done, W3.9). + +```bash +cd ~/projects/sde-api-scrapers/infrastructure +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +npm install -g aws-cdk # if not present +export AWS_PROFILE=sde-dev AWS_REGION=us-east-1 + +# run the offline suite first — it pins every injected env var and the dispatch trust +(cd .. && uv run pytest tests/ -q) # expect 316 passed (as of cf695f9) + +cdk synth -c environment=dev # must be clean +cdk diff -c environment=dev # review: new bucket, new role, new task def, alarm filter +cdk deploy --all -c environment=dev --require-approval never +``` + +Verify what was created: + +```bash +aws s3api head-bucket --bucket sde-cosmos-indexing-dev +aws s3api get-bucket-policy --bucket sde-cosmos-indexing-dev --query Policy --output text | python3 -m json.tool +aws iam get-role --role-name CosmosIndexingDispatchRole-dev --query 'Role.AssumeRolePolicyDocument' +aws ecs describe-task-definition --task-definition web_cosmos-scraper-dev \ + --query 'taskDefinition.containerDefinitions[0].[name,environment]' # name == WEB_COSMOSContainer, WEB_INDEX_NAME == sde-web-subset +``` + +- [x] Stacks deployed 2026-08-20 (outputs verified per the indexer tracker) +- [x] Bucket, dispatch role, and task family exist (`web_cosmos-scraper-dev:1`); trust policy names `arn:aws:iam::998871305517:role/indexing-helper-role` per the synthesized template — re-check live with the command above if desired +- [x] Spot-checked 2026-08-20: `web_cosmos-scraper-dev:1`, container `WEB_COSMOSContainer`, env + `WEB_INDEX_NAME=sde-web-subset`, `COSMOS_INDEX_BUCKET=sde-cosmos-indexing-dev`; cluster + `api-scrapers-cluster-dev` ACTIVE; `head-bucket` on `sde-cosmos-indexing-dev` OK + +### I.3 — Look up and send COSMOS the Fargate network values (**NS.3**) + +The cluster runs in the **default VPC**; the scheduled API tasks use public subnets and the VPC +default security group. COSMOS's `dispatch.py` sets `assignPublicIp: ENABLED`, so public subnets +are required (ECR/S3/SageMaker egress). + +```bash +export AWS_PROFILE=sde-dev AWS_REGION=us-east-1 +VPC=$(aws ec2 describe-vpcs --filters Name=isDefault,Values=true --query 'Vpcs[0].VpcId' --output text) +aws ec2 describe-subnets --filters Name=vpc-id,Values=$VPC Name=map-public-ip-on-launch,Values=true \ + --query 'Subnets[].SubnetId' --output text | tr '\t' ',' +aws ec2 describe-security-groups --filters Name=vpc-id,Values=$VPC Name=group-name,Values=default \ + --query 'SecurityGroups[0].GroupId' --output text +``` + +Looked up 2026-08-18 and **re-confirmed unchanged post-deploy 2026-08-20** (default VPC +`vpc-0265394c8c285afba`, all six subnets are `map-public-ip-on-launch`, one per AZ `us-east-1a`–`f`, +each with ~4,090 free IPs; default SG `sg-01817cfe4f3629986`; re-run the commands above if the VPC +changes). **This block is paste-ready for C.2:** + +``` +INDEXING_SUBNETS=subnet-0268b60265d9d6e87,subnet-0c29076fe7de10791,subnet-030a3a47fa10c76b2,subnet-0a6c6c437ed87dda3,subnet-09355979ab5496a50,subnet-0f3a7b40152e63be3 +INDEXING_SECURITY_GROUPS=sg-01817cfe4f3629986 +INDEXING_ECS_CLUSTER=api-scrapers-cluster-dev +INDEXING_TASK_FAMILY=web_cosmos-scraper-dev +INDEXING_CONTAINER_NAME=WEB_COSMOSContainer +SDE_INDEX_BUCKET=sde-cosmos-indexing-dev +INDEXING_DISPATCH_ROLE_ARN=arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev +``` + +- [x] Values looked up (above) — [x] **confirmed unchanged after I.2 (2026-08-20)**: same six + subnets, same default SG, cluster ACTIVE, task def env correct. Ready to paste into C.2. + +### I.4 — Prove the task runs end to end from the CLI (E2E.2 – E2E.9c, against `sde-web-subset`) + +Hand-write a 10-doc export, run the task, check the index. **Runnable now with no open prereqs** — +I.2 done 2026-08-20, and **both former prereqs passed 2026-08-20**: the I.1 image check (`:latest` = +merge SHA, tokenizer offline check green) and **OOB.3** (`sde-web-subset` verified to carry the +checked-in mapping — `version: keyword`, knn 768/BINARY incl. the nested full-text field; census: +59 docs across `astromaterials_data_system` (52) and `aurorasaurus_reporting_auroras_from_the_ground_up` +(7), **none carrying `version`**, 0 id-scheme mismatches, 0 duplicate URLs). Does **not** depend on +COSMOS. Read `first_run.md` first: on the **first** run of any collection +nothing is deleted (no document carries `version` yet, so the state scan is empty) and everything +is re-vectorized; the deletion guards only become live from run 2. The new **W2.13 id-collision +guard** runs before the state scan and refuses a collection the index already holds under ids the +pipeline would not mint (`id_scheme_collision`) or under duplicated ids (`duplicate_business_ids`); +`--allow-id-collision` overrides it and is recorded in `status.json` — never use it by default. + +```bash +export AWS_PROFILE=sde-dev AWS_REGION=us-east-1 +# 1) upload a fixture export (documents.jsonl first, manifest.json LAST) +aws s3 cp documents.jsonl s3://sde-cosmos-indexing-dev/curated_collections/verify_web/local-1/documents.jsonl +aws s3 cp manifest.json s3://sde-cosmos-indexing-dev/curated_collections/verify_web/local-1/manifest.json + +# 2) run the task manually with the same command override COSMOS will send +SUBNETS=...; SG=... # from I.3 +aws ecs run-task --cluster api-scrapers-cluster-dev --launch-type FARGATE \ + --task-definition web_cosmos-scraper-dev \ + --network-configuration "awsvpcConfiguration={subnets=[$SUBNETS],securityGroups=[$SG],assignPublicIp=ENABLED}" \ + --overrides '{"containerOverrides":[{"name":"WEB_COSMOSContainer","command":["python3","api_scraper.py","--source","WEB_COSMOS","--collection","verify_web","--target","test","--run-id","local-1"]}]}' + +# 3) watch it and read the result +aws logs tail /ecs/api-scrapers-dev --follow --log-stream-name-prefix web_cosmos-scraper # log group verified 2026-08-18 +aws s3 cp s3://sde-cosmos-indexing-dev/index_runs/verify_web/local-1/status.json - +aws s3 cp s3://sde-cosmos-indexing-dev/index_runs/verify_web/local-1/validation.json - +``` + +- [ ] E2E.3 `status.json` `state: succeeded`, `validation.json` full match +- [ ] E2E.4/5 `sde-web-subset` has 10 docs for `collection_key: verify_web`, knn hits return +- [ ] E2E.6 re-run → `changed: 0`; E2E.7a–c deletion guards; E2E.9/9a/9b/9c scope + tombstone checks (see tracker for exact assertions) +- [ ] Id-collision guard live: run against `gcn_circulars` (or `CODE_NASA_API`) → `status.json` `error: id_scheme_collision`, no SageMaker calls, nothing written; `status.json.id_collision_check` present on runs that pass the guard +- [ ] Tick these off in `Web Indexing - Task Plan & Tracking.md` + +### I.5 — Housekeeping (not blocking) + +- [x] **NS.7** — **closed 2026-08-20 by the merge to `develop` (PR #47)**: `deploy.yml`'s `test` job (incl. the rotation-matrix checksum) and `build-and-push` now run in CI for this code. numpy stays unpinned by decision (W0.1), guarded by the checksum test in CI. +- [ ] **NS.5** test/prod COSMOS account ids — only when those tiers exist +- [ ] **OOB.1** add `version: keyword` to live `sde-web` mapping — **cutover only** +- [ ] Cutover: flip `WEB_INDEX_NAME` `sde-web-subset` → `sde-web` (settings + task def) once E2E signed off +- [ ] Before any prod deploy: route `APIOpenSearchUploader` through the tier-capped client (today it uses bare `OPENSEARCH_ENDPOINT`; see "Known latent gap" above) + +--- + +## COSMOS + +### C.1 — Grant `sts:AssumeRole` on the dispatch role to `indexing-helper-role` (**NS.6**) — **APPLIED 2026-08-20** + +> **DONE 2026-08-20** — the `put-role-policy` below was run against `indexing-helper-role` and read +> back clean: inline policy **`CosmosIndexingDispatch-dev`**, one statement, +> `sts:AssumeRole` → `arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev`. The dispatch +> role's live trust policy names `arn:aws:iam::998871305517:role/indexing-helper-role`, and +> `aws iam simulate-principal-policy` evaluates the pair as **`allowed`**. Because the grant is on +> the *role*, it covers both COSMOS boxes that use `indexing-helper-role` — staging +> (`i-08f9b2175b70fa05c`, where this loop runs) and production (`i-02b3d3e1ac0671952`). +> +> **Still open:** the on-host confirmation. SSM could not reach either box (see the SSM note at the +> top of this file), so run it over SSH instead: +> +> ```bash +> ssh staging_cosmos # ec2-user@18.215.146.207, ~/.ssh/sde-indexing-helper-staging.pem +> aws sts assume-role --region us-east-1 \ +> --role-arn arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev \ +> --role-session-name cosmos-preflight --query Credentials.Expiration +> ``` +> +> An expiration timestamp = C.1 fully verified. While on the host, `sudo systemctl restart +> amazon-ssm-agent` is worth running — C.2's instructions assume SSM works. + +**The role exists as of 2026-08-20 (I.2 done) — do this now.** Don't wait for Phase 9's +`preflight_aws`, which only *checks* this grant. Same-account trust is satisfied only by the +identity policy, so without this every dispatch fails with `AccessDenied` on `AssumeRole`. +No S3 permissions are needed on our side: the bucket policy names our role directly (and `indexing-helper-role` already carries `AmazonS3FullAccess` + `indexing-helper-s3-access`; as of 2026-08-18 it had **no inline policies** and no `sts:AssumeRole` anywhere — the C.1 policy below is what changed that on 2026-08-20, alongside an `AmazonSSMManagedInstanceCore` attach for SSM). + +```bash +export AWS_PROFILE=sde-dev AWS_REGION=us-east-1 +cat > cosmos-indexing-dispatch.json <<'JSON' +{ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "AssumeCosmosIndexingDispatchRoleDev", + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev" + }] +} +JSON +aws iam put-role-policy --role-name indexing-helper-role \ + --policy-name CosmosIndexingDispatch-dev \ + --policy-document file://cosmos-indexing-dispatch.json + +# verify from the COSMOS STAGING host itself (`ssh staging_cosmos` — SSM is not registered yet): +aws sts assume-role --role-arn arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev \ + --role-session-name cosmos-preflight --query 'Credentials.Expiration' +``` + +- [x] Inline policy attached (2026-08-20, read back; `simulate-principal-policy` → `allowed`) +- [x] `assume-role` from the COSMOS **staging** host succeeds — run on `ec2-user@STAGING` + 2026-08-20, returned `Credentials.Expiration = 2026-08-20T22:53:59+00:00`. **C.1 / NS.6 DONE.** + +### C.2 — Set the indexing env vars on the staging host (**NS.3, our half**) + +Env files are hand-maintained per host (`.envs/.production/.django` on the Django/Celery host, +per `sde_collections/DEPLOYMENT.md`). Every dispatch-gating setting defaults blank/off, so nothing +dispatches until these are present. `INDEXING_CONTAINER_NAME` already defaults to `WEB_COSMOSContainer` +and `AWS_REGION` to `us-east-1` (set it explicitly if the host is ever elsewhere); `launchType=FARGATE` +and `assignPublicIp=ENABLED` are hard-coded in `dispatch.py`, not settings. + +```bash +# on the COSMOS STAGING host: `ssh staging_cosmos` (ec2-user@18.215.146.207) +# — SSM (`aws ssm start-session --target i-08f9b2175b70fa05c`) works only once the agent registers; +# see the SSM note at the top of this file. Do NOT use i-0178c998e868792d7 +# (`COSMOS_Staging_Refresh`) — it has no instance profile, so dispatch cannot work from it. +sudo -e /path/to/.envs/.production/.django # append: +SDE_INDEX_BUCKET=sde-cosmos-indexing-dev +INDEXING_ECS_CLUSTER=api-scrapers-cluster-dev +INDEXING_TASK_FAMILY=web_cosmos-scraper-dev +INDEXING_CONTAINER_NAME=WEB_COSMOSContainer +INDEXING_DISPATCH_ROLE_ARN=arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev +INDEXING_SUBNETS= +INDEXING_SECURITY_GROUPS= +INDEX_POLL_ENABLED=true + +# 1) migrate — the poll_index_runs / poll_scrape_jobs beat rows are (re)written and enabled by a +# post_migrate receiver (sde_collections/signals.py); the production start script does NOT migrate, +# so without this step INDEX_POLL_ENABLED=true changes nothing (DEPLOYMENT.md step 4) +docker compose -f production.yml run --rm django python manage.py migrate --noinput +# 2) restart the services that read settings +docker compose -f production.yml up -d --force-recreate django celeryworker celerybeat +docker compose -f production.yml run --rm django python manage.py shell -c \ + "from django.conf import settings as s; print(s.INDEXING_DISPATCH_ROLE_ARN, s.INDEXING_SUBNETS, s.INDEX_POLL_ENABLED)" +``` + +- [ ] Vars present in the running containers; `PeriodicTask` "Poll index runs (every 2 min)" exists with `enabled=True` (`python manage.py shell -c "from django_celery_beat.models import PeriodicTask; print(list(PeriodicTask.objects.values_list('name','enabled')))"`). Note the flag gates only the beat row — a manual `poll_index_runs.delay()` runs regardless. + +### C.3 — Pre-flight from the COSMOS side (before firing a real curation) + +```bash +# S3 probes — run FROM THE COSMOS STAGING HOST (`ssh staging_cosmos`, its role). The bucket policy grants only s3:PutObject on +# curated_collections/* and s3:GetObject on index_runs/* (no ListBucket), but in dev the same-account +# indexing-helper-role also carries AmazonS3FullAccess, so these are smoke tests, not permission tests: +# expect the put and the get to succeed; do NOT expect a DENY on the read-back, and `aws s3 ls` may +# 403 only if the identity policy is ever narrowed. +echo test | aws s3 cp - s3://sde-cosmos-indexing-dev/curated_collections/_preflight/x/probe.txt +aws s3 cp s3://sde-cosmos-indexing-dev/curated_collections/_preflight/x/probe.txt - +aws s3api list-objects-v2 --bucket sde-cosmos-indexing-dev --prefix index_runs/ --max-keys 5 # ok or AccessDenied — both fine + +# dry dispatch from a Django shell against a scratch collection +docker compose -f production.yml run --rm django python manage.py shell -c \ + "from sde_collections.models.collection import Collection; from sde_collections.indexing.dispatch import run_index_task; \ + c=Collection.objects.get(config_folder='verify_web'); print(run_index_task(c, 'test', 'preflight-1'))" +``` + +- [ ] `taskArn` returned; task visible with `aws ecs list-tasks --cluster api-scrapers-cluster-dev --family web_cosmos-scraper-dev` +- [ ] Optional: run `python manage.py preflight_aws` once Phase 9 lands it + +### C.4 — Closed-loop verification (their **E2E.10**, our Phase 7 last done-when) + +1. Pick a small, non-collision collection (not one of the 12 id-scheme-collision collections — e.g. **not** `gcn_circulars` or `CODE_NASA_API`). The indexer now refuses those itself (`id_scheme_collision` / `duplicate_business_ids`), which COSMOS would see as `INDEXING_FAILED_ON_TEST` — a correct refusal, not a bug. +2. In the admin, set `workflow_status` → **Curated**. +3. Confirm: `IndexDispatch` row created; export objects under `curated_collections/{cf}/{run_id}/` with `manifest.json` last; ECS task running; status moves to `TEST_INDEXING`. +4. Wait for `index_runs/{cf}/{run_id}/status.json`; poller keeps the collection in `TEST_INDEXING` and posts the `validation.json` summary to Slack (channel is whatever `SLACK_WEBHOOK_URL` is bound to — COSMOS sends no `channel` field; `#sde-data-curation` is the expected binding). +5. Set **QC Perfect** → confirm a prod dispatch fires (`--target prod`, still landing in `sde-web-subset` until cutover) and status lands on `PROD_PERFECT`. +6. Failure path: hand-corrupt a run (e.g. delete `manifest.json` before dispatch) → `INDEXING_FAILED_ON_TEST`. + +```bash +# useful while watching +docker compose -f production.yml run --rm django python manage.py shell -c \ + "from sde_collections.models.indexing import IndexDispatch; print(list(IndexDispatch.objects.order_by('-dispatched_at').values()[:3]))" +aws s3 ls s3://sde-cosmos-indexing-dev/index_runs/ --recursive | tail +``` + +- [ ] Tick "Closed loop verified against dev" in `IMPLEMENTATION_PLAN.md` Phase 7 done-when +- [ ] Update `IMPLEMENTATION_PLAN.md` cross-repo status: NS.3 → `[x]` (NS.2 and NS.6 both flipped 2026-08-20) + +### C.5 — Later / not blocking + +- [ ] Phase 9: `preflight_aws` gains the `SDE_INDEX_BUCKET` and `sts:AssumeRole` checks (verifies C.1, doesn't replace it) +- [ ] The 12 id-scheme-collision collections: the indexer's **guard landed** (`cf695f9`, W2.13) and refuses them at run time, so onboarding one is safe-but-futile (it fails fast, no spend). Keep them out of the flow until the *repair* (`FINDING_id_scheme_collision.md` §9) lands; expect `INDEXING_FAILED_ON_TEST` if one slips through +- [ ] Cutover needs **no COSMOS change** — it is the indexer's `WEB_INDEX_NAME` flip + +--- + +## Full workflow — COSMOS ↔ crawler ↔ indexer, end to end + +Every hop is a **file in S3 (or a JSON drop) plus a poller** — no callbacks, no `describe_tasks`. +Function/script names are the real ones in each repo. Statuses are `WorkflowStatusChoices` unless +marked `reindexing_status`. + +``` + COSMOS (Django + Celery) crawl4ai host (i-0b6a…) sde-api-scrapers (ECS Fargate) + ──────────────────────── ───────────────────── ────────────────────────────── + [1] status → READY_FOR_ENGINEERING + handle_workflow_status_change + └─ dispatch_scrape_job.delay ──SSM──▶ jobs/incoming/{cf}.json + watcher → crawl → S3 + [2] poll_scrape_jobs (5 min) ◀───S3────── scraped_collections/{cf}.json + └─ ingest_scraped_collection failure_logs/{cf}_failures_summary.json + → SCRAPING_SUCCESSFUL + └─ migrate_dump_to_delta… + → READY_FOR_CURATION + [3] curator works in COSMOS UI + [4] status → CURATED + handle_workflow_status_change + └─ promote_to_curated() + └─ index_collection_to_test.delay + export_curated_to_s3 ──S3──────────────────────────────▶ curated_collections/{cf}/{run_id}/documents.jsonl + curated_collections/{cf}/{run_id}/manifest.json (last) + run_index_task ──sts:AssumeRole → ecs:RunTask ────────▶ web_cosmos-scraper-dev + IndexDispatch row; → TEST_INDEXING python3 api_scraper.py --source WEB_COSMOS + --collection {cf} --target test --run-id {run_id} + WebPipeline.run() (steps below) + [5] poll_index_runs (2 min) ◀───S3────────────────────────────── index_runs/{cf}/{run_id}/status.json (last, always) + fetch_run_status / fetch_validation_report index_runs/{cf}/{run_id}/validation.json (test only) + succeeded+test → stays TEST_INDEXING, + send_indexing_validation_report → Slack #sde-data-curation + [6] curator reads report, sets QC_PERFECT / QC_MINOR + handle_workflow_status_change + └─ index_collection_to_prod.delay (same export → RunTask, --target prod) + → PRODUCTION_INDEXING + [7] poll_index_runs: succeeded+prod → PROD_PERFECT / PROD_MINOR + failed / unknown state / stall (INDEX_STALL_TIMEOUT_HOURS=6) → INDEXING_FAILED_ON_TEST|PROD +``` + +### Sequence diagram + +```mermaid +sequenceDiagram + autonumber + actor Eng as Engineer / Curator + participant Cosmos as COSMOS (Django + Celery) + participant SSM as AWS SSM + participant Crawler as crawl4ai host + participant CrawlS3 as S3 crawler bucket + participant IdxS3 as S3 sde-cosmos-indexing-dev + participant STS as STS / ECS + participant Idx as WEB_COSMOS task (Fargate) + participant AOSS as AOSS sde-web-subset + SageMaker + participant Slack as Slack (sde-data-curation) + + rect rgb(235, 245, 235) + Note over Eng,CrawlS3: Phase 3–4 · scrape + Eng->>Cosmos: workflow_status = READY_FOR_ENGINEERING + Cosmos->>Cosmos: handle_workflow_status_change → dispatch_scrape_job.delay + Cosmos->>SSM: send_command(AWS-RunShellScript) — write jobs/incoming/{cf}.json (.tmp → mv) + Cosmos->>Cosmos: ScrapeDispatch(dispatched_at) + SSM->>Crawler: run script on i-0b6a61d95888886f4 + Crawler->>Crawler: watcher picks up job → crawl (≤ MAX_PAGES_CAP) + Crawler->>CrawlS3: scraped_collections/{cf}.json + failure_logs/{cf}_failures_summary.json + loop poll_scrape_jobs every 5 min (SCRAPE_POLL_ENABLED) + Cosmos->>CrawlS3: results_ready? summary.LastModified > dispatched_at + CrawlS3-->>Cosmos: summary + documents + end + alt fresh results + Cosmos->>Cosmos: ingest_scraped_collection (CAS) → SCRAPING_SUCCESSFUL → DumpUrl + Cosmos->>Cosmos: migrate_dump_to_delta… → DeltaUrl → READY_FOR_CURATION + Cosmos->>Slack: ingest summary + else zero pages / stall > 24 h + Cosmos->>Cosmos: SCRAPING_FAILED + end + end + + Eng->>Cosmos: curate in UI (patterns, exclusions, titles) + + rect rgb(232, 242, 247) + Note over Eng,Slack: Phase 7 · test indexing + Eng->>Cosmos: workflow_status = CURATED + Cosmos->>Cosmos: promote_to_curated() → index_collection_to_test.delay + Cosmos->>Cosmos: _dispatch_index_run: run_id = _mint_run_id() + Cosmos->>IdxS3: export_curated_to_s3 → curated_collections/{cf}/{run_id}/documents.jsonl + Cosmos->>IdxS3: manifest.json (written LAST, document_count exact) + Cosmos->>STS: sts:AssumeRole CosmosIndexingDispatchRole-dev + STS-->>Cosmos: temp credentials + Cosmos->>STS: ecs:RunTask web_cosmos-scraper-dev, override: python3 api_scraper.py --source WEB_COSMOS --collection {cf} --target test --run-id {run_id} + STS-->>Cosmos: taskArn + Cosmos->>Cosmos: IndexDispatch(run_id, target=test, task_arn) → TEST_INDEXING + STS->>Idx: start container WEB_COSMOSContainer + Idx->>IdxS3: load_manifest + Idx->>AOSS: ensure_index → probe_scope + Idx->>AOSS: check_id_collisions (W2.13) — refuse id_scheme_collision / duplicate_business_ids before any spend + Idx->>AOSS: fetch_index_state(filter) + assert_owned + Idx->>IdxS3: stream_ids (id-only pre-pass) + Idx->>Idx: check_export_completeness · evaluate_deletions (cap 5000, then ratio > 0.90) — before any spend + Idx->>IdxS3: stream_documents + Idx->>AOSS: to_web_document → skip unchanged version → vectorize_documents_chunkwise → index_batch + Idx->>AOSS: tombstone_batch (public_visibility=false) — only if upsert complete + Idx->>AOSS: validate_run (test only) + Idx->>IdxS3: index_runs/{cf}/{run_id}/validation.json + Idx->>IdxS3: index_runs/{cf}/{run_id}/status.json — LAST, always, incl. on failure + loop poll_index_runs every 2 min (INDEX_POLL_ENABLED) + Cosmos->>IdxS3: fetch_run_status(cf, run_id) + IdxS3-->>Cosmos: status.json or nothing yet + end + alt state == succeeded (target test) + Cosmos->>IdxS3: fetch_validation_report + Cosmos->>Slack: send_indexing_validation_report — collection stays TEST_INDEXING + else failed · unknown state · stall > 6 h + Cosmos->>Cosmos: INDEXING_FAILED_ON_TEST + end + end + + rect rgb(247, 240, 228) + Note over Eng,IdxS3: Phase 7 · prod indexing + Eng->>Cosmos: workflow_status = QUALITY_CHECK_PERFECT / QUALITY_CHECK_MINOR (from the report) + Cosmos->>Cosmos: index_collection_to_prod.delay → new run_id + Cosmos->>IdxS3: export again (documents.jsonl, then manifest.json) + Cosmos->>STS: AssumeRole → RunTask … --target prod --run-id {run_id2} + Cosmos->>Cosmos: IndexDispatch → PRODUCTION_INDEXING + STS->>Idx: start task + Idx->>AOSS: same pipeline (still sde-web-subset until cutover — endpoints tier-capped) + Idx->>IdxS3: index_runs/{cf}/{run_id2}/status.json + loop poll_index_runs every 2 min + Cosmos->>IdxS3: fetch_run_status + end + alt succeeded + Cosmos->>Cosmos: PROD_PERFECT (or PROD_MINOR if previous status was QC_MINOR) + else failed · unknown · stall + Cosmos->>Cosmos: INDEXING_FAILED_ON_PROD + end + end +``` + +### Step by step + +**[1] Scrape dispatch (COSMOS → crawler, Phase 3)** +- Trigger: `Collection.workflow_status` set to `READY_FOR_ENGINEERING` (or + `reindexing_status` = `REINDEXING_NEEDED_ON_DEV` for a re-scrape) → `post_save` → + `handle_workflow_status_change` (`sde_collections/models/collection.py`). +- `sde_collections/tasks.py::dispatch_scrape_job` → `scraping/ssm_dispatch.py::send_job_to_crawler`: + builds the job JSON (`scraping/job_builder.py::build_job_json`; a request above the hard-coded `MAX_PAGES_CAP = 100_000` is rejected with `ValueError`, not clamped), + `ssm.send_command(AWS-RunShellScript)` to `CRAWLER_INSTANCE_ID`, writing + `{CRAWLER_INBOX_PATH}/{cf}.json` via a `.tmp` + `mv` so the watcher never reads a partial file. +- Records a `ScrapeDispatch` row (`ssm_command_id`, `dispatched_at`) — the poller's freshness + reference. Failure to send → `SCRAPING_FAILED`. +- Manual equivalent: `python manage.py dispatch_scrape --collection `. + +**[2] Scrape results (crawler → S3 → COSMOS, Phase 4)** +- Crawler writes to `SDE_S3_BUCKET` (`sdecrawlerstack-crawlbucket…`): + `scraped_collections/{cf}.json`, `failure_logs/{cf}_failures.jsonl`, + `failure_logs/{cf}_failures_summary.json`. +- Beat: `poll_scrape_jobs` every 5 min (`SCRAPE_POLL_ENABLED`). `scraping/s3_results.py::results_ready` + accepts a summary only if its `LastModified` is after `ScrapeDispatch.dispatched_at`; past + `SCRAPE_STALL_TIMEOUT_HOURS` (24) with nothing fresh → `SCRAPING_FAILED`. +- `ingest_scraped_collection` claims the collection with an atomic status CAS → `SCRAPING_SUCCESSFUL`, + loads `DumpUrl`s (zero pages → `SCRAPING_FAILED`), then + `migrate_dump_to_delta_and_handle_status_transistions` → `DeltaUrl`s → `READY_FOR_CURATION` + (re-scrape: `REINDEXING_FINISHED_ON_DEV` → `REINDEXING_READY_FOR_CURATION`) and posts the + ingest summary to Slack. + +**[3] Curation** — human step in the COSMOS UI (patterns, exclusions, titles, doc types). + +**[4] Test-index hand-off (COSMOS → indexer, Phase 7 / W4)** +- Trigger: `CURATED` → `handle_workflow_status_change` → `promote_to_curated()` (Delta → Curated + URLs) → `index_collection_to_test.delay`. +- `tasks.py::_dispatch_index_run` (shared by test and prod): + 1. `_mint_run_id()`. + 2. `indexing/export.py::export_curated_to_s3(collection, target, run_id)` — iterates + `CuratedUrl.objects.filter(collection=c).exclude(excluded=True)`, spools JSONL to a temp + file, uploads `documents.jsonl`, then `manifest.json` **last** (`document_count` exact). + Bucket: `SDE_INDEX_BUCKET` = `sde-cosmos-indexing-dev`. + 3. `indexing/dispatch.py::run_index_task(collection, target, run_id)` — + `sts.assume_role(INDEXING_DISPATCH_ROLE_ARN)` → `ecs.run_task(cluster=INDEXING_ECS_CLUSTER, + taskDefinition=INDEXING_TASK_FAMILY, overrides.containerOverrides[name=WEB_COSMOSContainer, + command=["python3","api_scraper.py","--source","WEB_COSMOS","--collection",cf,"--target", + target,"--run-id",run_id]], networkConfiguration from INDEXING_SUBNETS/SECURITY_GROUPS)` + → returns `taskArn`. + 4. `IndexDispatch` row (`collection, run_id, target, task_arn, dispatched_at, + previous_workflow_status`) → status `TEST_INDEXING`. + - Any exception in 2–3 (including an empty curated set — `document_count == 0` raises before export) → `INDEXING_FAILED_ON_TEST`, nothing recorded. + +**Indexer run (`sde-api-scrapers`, `api_scraper.py::_run_web_cosmos` → `web/web_pipeline.py::WebPipeline.run()`)** +- Reads `COSMOS_INDEX_BUCKET`, `WEB_INDEX_NAME` (=`sde-web-subset` until cutover), both AOSS + endpoints (probe/validate client tier-capped by `--target`; the uploader's write path uses bare + `OPENSEARCH_ENDPOINT` — same collection in dev), deletion knobs — all injected by the task definition. +- Order is load-bearing: + 1. `cosmos_source.load_manifest` (`curated_collections/{cf}/{run_id}/manifest.json`). + 2. `ensure_index` — explicit mapping from `web/index_mappings/sde_web.json` (`version: keyword`, binary knn). + 3. `scope.probe_scope` — proves `WebIndexScope(cf).filter_query` isolates this collection + (else `scope_filter_ineffective`). + 3b. `id_collision.check_id_collisions` (**W2.13**, `cf695f9`) — refuses if the index already holds + this collection under ids the pipeline would not mint (`id_scheme_collision`) or under + duplicated business ids (`duplicate_business_ids`); the upsert would insert, not update, and + silently double the collection. Skipped when the index is absent (first run); + `--allow-id-collision` overrides and records the waiver in `status.json.id_collision_check`. + 4. `uploader.fetch_index_state(filter_query)` + `scope.assert_owned(…, "state_scan")` + (foreign id → `foreign_documents_in_scan`, zero deletes). + 5. Id-only pre-pass over the export (`cosmos_source.stream_ids`) → + `deletion_guard.check_export_completeness` (line count / dupes vs `document_count` → + `export_incomplete`, deletions skipped) and `evaluate_deletions` (count > `WEB_DELETION_ABORT_MAX` + (5000) → `deletion_budget_exceeded`, checked first; then ratio **> 0.90** (exactly 0.90 passes) → + `deletion_threshold_exceeded`). `assert_owned(…, "export_ids")` also runs on the id pre-pass. + All of this **before** any SageMaker spend. + 6. `_index_documents`: `cosmos_source.stream_documents` → `web_processor.to_web_document` + (mints `id=/SDE/{cf}/|{url}`, content-hash `version`, drops `tdamm_tag`) → skip unchanged + versions → `SageMakerVectorizer.vectorize_documents_chunkwise` → `assert_owned(…, "upsert_batch")` + → `uploader.index_batch`. + 7. `uploader.tombstone_batch` (sets `public_visibility: false`, reversible) — only if the upsert + completed, and only after `assert_owned(…, "deletion_candidates")`. + 8. `--target test` only: `validate.validate_run` → `index_runs/{cf}/{run_id}/validation.json` + (count + title diff vs manifest; report, never a gate). + 9. `status.json` written **last and unconditionally**, incl. on failure + (`state`, counts, `deletion_ratio`, `deletion_mode`, machine-readable `error`). Exit code + 0 iff `state == succeeded`. +- **First run of a collection** (`first_run.md`): no document carries `version` yet, so the state + scan is empty, the deletion-candidate set is structurally empty, and every document is + re-vectorized (full SageMaker cost). Run 1 writes `version`, which is what arms the deletion + guards for run 2 onward — treat run 2 as the first one where the safety machinery matters. +- `--reconcile` (operator-only, never dispatched by COSMOS): scans without the `version` filter, + writes `reconcile.json` listing orphans — report, no deletes. + +**[5] Poll test result (COSMOS)** +- Beat: `poll_index_runs` every 2 min (`INDEX_POLL_ENABLED` — enforced on the `PeriodicTask.enabled` row, written at `post_migrate`). For each collection in + `TEST_INDEXING`/`PRODUCTION_INDEXING` with an open `IndexDispatch`: + `indexing/run_status.py::fetch_run_status(cf, run_id)` reads `index_runs/{cf}/{run_id}/status.json`. +- `None` → still running; past `INDEX_STALL_TIMEOUT_HOURS` (6) → `INDEXING_FAILED_ON_*`. +- `succeeded` + `test` → collection **stays** `TEST_INDEXING`; `fetch_validation_report` → + `slack_utils.send_indexing_validation_report` posts to `#sde-data-curation`. +- `failed` or any unknown `state` → `INDEXING_FAILED_ON_TEST`. `IndexDispatch.completed_at` set. + +**[6] QC → prod hand-off (COSMOS)** +- Curator reads the Slack report and sets `QUALITY_CHECK_PERFECT` or `QUALITY_CHECK_MINOR` + → `handle_workflow_status_change` → `index_collection_to_prod.delay` → same + `_dispatch_index_run` with `target="prod"` (new `run_id`, fresh export, `--target prod`) → + `PRODUCTION_INDEXING`. (Until cutover the prod run still lands in `sde-web-subset`; in dev every + endpoint env var resolves to the dev collection, so a dev dispatch cannot reach prod AOSS.) + +**[7] Poll prod result (COSMOS)** +- `poll_index_runs`: `succeeded` + `prod` → `PROD_PERFECT`, or `PROD_MINOR` if + `previous_workflow_status` was `QUALITY_CHECK_MINOR`; failure/stall → `INDEXING_FAILED_ON_PROD`. +- Re-index later: `reindexing_status` = `REINDEXING_NEEDED_ON_DEV` restarts at [1]; + `REINDEXING_CURATED` → `promote_to_curated()` (then the curator sets `CURATED` to re-run [4]). + +### Where each side's guarantees live + +| Concern | Owner | Mechanism | +|---|---|---| +| Stale crawler results | COSMOS | `ScrapeDispatch.dispatched_at` vs S3 `LastModified` | +| Stale index results | structural | `run_id` namespaces every artifact — no freshness rule needed | +| Partial export read as complete | COSMOS writes manifest **last**; indexer checks count | `export_incomplete` → deletions skipped | +| Mass deletion / mis-scope | indexer | scope probe, ownership assertion at 4 boundaries, absolute cap + ratio (> 0.90) | +| Wasted SageMaker spend | indexer | all guards run before vectorization | +| Reversibility | indexer | tombstones (`public_visibility: false`), never hard deletes | +| Which index | indexer | `WEB_INDEX_NAME` in the task def (`sde-web-subset` now; flip = cutover) | +| Silent duplication (id-scheme drift) | indexer | W2.13 `check_id_collisions` — `prefix` must-not on `id` + `min_doc_count: 2` agg, both fail-fast; `--allow-id-collision` is an audited override | +| Who may dispatch | indexer IAM | dispatch role trusts `indexing-helper-role` only; RunTask limited to the `web_cosmos-scraper-{env}` family + cluster | diff --git a/LOCAL_VERIFICATION_GUIDE.md b/LOCAL_VERIFICATION_GUIDE.md new file mode 100644 index 00000000..86c5a186 --- /dev/null +++ b/LOCAL_VERIFICATION_GUIDE.md @@ -0,0 +1,674 @@ +# COSMOS Local Verification Guide — Phases 0–7 + +This guide verifies the rewired COSMOS pipeline (branch `cosmos-rewiring`, Phases 0–7) by +**walking one real collection through the entire workflow on your machine**: +*Aurorasaurus — Reporting Auroras from the Ground Up* +(`config_folder: aurorasaurus_reporting_auroras_from_the_ground_up`). + +Why this collection: the crawl4ai scraper **already crawled it for real** — its output (25 +documents from `aurorasaurus.org`) sits in the dev S3 bucket. That lets the demo ingest *genuine +scraped data* through the *genuine production code path*, no mocks. Only one thing stays +simulated: the WEB_COSMOS indexer's responses. Its AWS stacks *are* deployed in the dev account +(the `sde-cosmos-indexing-dev` bucket, the `CosmosIndexingDispatchRole-dev` role, and the +`web_cosmos-scraper-dev` task family) — simulating its replies is a choice of this local +walkthrough, which runs COSMOS on your machine and deliberately starts no real Fargate tasks. + +Each step of the walkthrough has the same shape: + +- **Where we are** — what this hop means in the curation workflow, and which phase built it. +- **Do this** — exact commands / clicks. +- **Expect this** — the observed outcome (this walkthrough was executed successfully on + 2026-08-14; every expected output below is what actually happened). +- **Phase tests** — the pytest suite that locks the behavior in. + +*No dev-AWS access?* Appendix A has a fully-offline variant of the ingest using mocked S3. + +--- + +## The workflow you are about to walk + +``` +Research in Progress + │ curator finishes research, sets… + ▼ +Ready for Engineering ──────► COSMOS builds a job JSON and SSM-sends it (Phase 3) + │ to the crawl4ai crawler on EC2 + ▼ + [crawler runs; writes results + a completion summary to S3] + │ + ▼ +Scraping Successful ────────► COSMOS polls S3, ingests documents into (Phase 4) + │ DumpUrls, then migrates them to DeltaUrls + ▼ (inference is dormant — Phase 2 — so the +Ready for Curation migration runs immediately) + │ curator reviews/edits the deltas in the UI, sets… + ▼ +Curated ────────────────────► promote_to_curated(): deltas become (Phase 5) + │ CuratedUrls; COSMOS exports them to S3 and + ▼ fires the WEB_COSMOS indexer (test target) (Phase 7) +Test Indexing ──────────────► poller reads the indexer's status.json; + │ validation report posts to Slack + │ curator QCs the test index, sets… + ▼ +QC: Perfect / QC: Minor ────► same hand-off, prod target (Phase 7) + ▼ +Production Indexing ────────► poller resolves; final status mirrors QC: + ▼ +Prod: Perfect / Prod: Minor Issues ← the finish line +``` + +Failure at any dispatch/poll lands on a red status (**Scraping Failed**, **Indexing Failed on +Test/Prod**) instead of raising — which this guide uses deliberately: with the indexer's AWS +settings blank, its two dispatch hops fail *gracefully*, and that graceful failure is itself the +proof the trigger fired. Phase 0 supplies the shared settings/credentials plumbing, Phase 1 the +six new statuses, Phase 6 removed the old Sinequa machinery entirely. + +--- + +## 0. Setup + +### Prerequisites + +- Docker Desktop running. +- Repo on branch `cosmos-rewiring`. +- A local database restored from production (the Aurorasaurus collection must exist — checked in + Step A below). +- For the real-S3 ingest: `aws sso login --profile sde-dev` (account 998871305517). Everything + else works without it. + +### Environment — `.envs/.local/.django` + +All pipeline vars already exist in this file. For this walkthrough the AWS-facing ones stay +**blank** and the pollers **off**: + +| Var | Value | Why | +|---|---|---| +| `SLACK_WEBHOOK_URL` | any non-empty dummy, e.g. `http://localhost/dummy` | **Required to boot** (no default). Dummy → every Slack post prints a caught error and continues. Real webhook → messages actually post on every status change. | +| `SDE_S3_BUCKET`, `CRAWLER_INSTANCE_ID` | blank | The scrape-dispatch hop then fails gracefully (Step 3's wiring proof). The real bucket is injected per-command in Step 4, not set here. | +| `SDE_INDEX_BUCKET`, `INDEXING_*` | blank (keep `INDEXING_CONTAINER_NAME` default) | The indexing hops then fail gracefully (Steps 5 & 7). Deliberate: the dev stacks are live, and a local run should not export to their bucket or start real Fargate tasks. | +| `SCRAPE_POLL_ENABLED`, `INDEX_POLL_ENABLED`, `INFERENCE_ENABLED` | `False` | With blank buckets the pollers would only log S3 errors every 2–5 min; the inference-off state is itself verified in Step 2. | +| `SDE_AWS_ACCESS_KEY_ID`, `SDE_AWS_SECRET_ACCESS_KEY` | blank | Keeps `get_boto3_session()` on the default credential chain — which is exactly how Step 4 injects short-lived SSO credentials. | + +### Build and start + +```bash +docker-compose -f local.yml build +docker-compose -f local.yml up -d # django, postgres, redis, celeryworker, celerybeat, flower +docker-compose -f local.yml run --rm django python manage.py createsuperuser # first time only +``` + +Migrations run automatically on startup. Confirm the rewiring migrations are in: + +```bash +docker-compose -f local.yml run --rm django python manage.py showmigrations sde_collections | tail -3 +# [X] 0078_alter_collection_workflow_status_and_more ← statuses 21–26 (Phase 1) +# [X] 0079_scraperconfigoverride_scrapedispatch ← Phase 3 models +# [X] 0080_indexdispatch ← Phase 7 model +``` + +### URLs + +> **Gotcha:** the django log says `Starting development server at http://0.0.0.0:8000/` — that is +> the address *inside the container*. On your host the app is on **8001** (`local.yml` maps +> `8001:8000`; host port 8000 usually belongs to the inference API, which answers +> `{"detail": "Not Found"}` if you hit it by mistake). First visit redirects to the login page — +> sign in with your superuser. + +| What | URL | +|---|---| +| Collection list (main UI) | `http://localhost:8001/` | +| Collection detail | `http://localhost:8001//` | +| Django admin | `http://localhost:8001/admin/` | +| Flower (Celery monitor) | `http://localhost:5555/` | + +### Running the tests + +> **Important — test isolation from the live worker:** `config/settings/test.py` forces +> `CELERY_BROKER_URL=memory://` (both the setting *and* the env var — Celery gives the env var +> precedence). Without it, tests that change workflow statuses publish real task messages onto +> the same Redis your celeryworker consumes, and the worker executes them against your **local +> database** — test-DB collection ids can collide with real rows and silently flip their +> statuses. If you ever see a burst of `dispatch_scrape_job` / `DoesNotExist` noise in the +> worker log right after a pytest run, check that override is still in place. + +```bash +# Everything in sde_collections (expect: 308 passed): +docker-compose -f local.yml run --rm django pytest sde_collections/tests/ + +# Just the rewiring suites (expect: 129 passed = 4 + 74 + 7 + 10 + 15 + 19): +docker-compose -f local.yml run --rm django pytest \ + sde_collections/tests/test_aws_utils.py \ + sde_collections/tests/test_workflow_status_triggers.py \ + sde_collections/tests/test_inference_flag.py \ + sde_collections/tests/test_scrape_dispatch.py \ + sde_collections/tests/test_scrape_ingest.py \ + sde_collections/tests/test_indexing_dispatch.py +``` + +### Shell for the paste-in snippets + +Every Python snippet below runs in: + +```bash +docker-compose -f local.yml run --rm django python manage.py shell_plus +``` + +--- + +## Step A — Meet the demo collection (baseline) + +**Where we are.** Before touching anything, establish what exists: the collection in your local +DB, and its finished crawl in the dev S3 bucket. + +**Do this** — in `shell_plus`: + +```python +from sde_collections.models.collection import Collection +c = Collection.objects.get(config_folder="aurorasaurus_reporting_auroras_from_the_ground_up") +print(c.id, "|", c.name) +print("status:", c.get_workflow_status_display()) +print("dump:", c.dump_urls.count(), "| delta:", c.delta_urls.count(), "| curated:", c.curated_urls.count()) +``` + +**Expect this:** the collection exists (id `1431` on the 2026-08 restore) with **25 curated +URLs** — it has been through curation before, which makes the Step 4 diff meaningful. Its status +may be anything; the walkthrough sets what it needs at each step, and the whole run is +**repeatable** (re-ingesting replaces dumps and re-diffs; nothing accumulates). + +And, if you have `sde-dev` access, confirm the crawl output (the completion marker is the +`_failures_summary.json` — the crawler writes it only at the end of a completed run): + +```bash +aws s3 ls s3://sdecrawlerstack-crawlbucket0d63eba8-lhkxqnh8ophy/scraped_collections/ --profile sde-dev | grep aurorasaurus +aws s3 ls s3://sdecrawlerstack-crawlbucket0d63eba8-lhkxqnh8ophy/failure_logs/ --profile sde-dev | grep aurorasaurus +# aurorasaurus_reporting_auroras_from_the_ground_up.json (25 documents, 2026-08-13) +# aurorasaurus_reporting_auroras_from_the_ground_up_failures_summary.json (the completion marker) +``` + +--- + +## Step 0 — Foundations hold (Phase 0) + +**Where we are.** Phase 0 added the `SDE_*` settings block (every var with a safe default) and +`get_boto3_session()` — the single credential entry point for all pipeline AWS code: explicit +`SDE_AWS_*` keys if both are set, otherwise the default chain (instance role in AWS, env vars +locally). The fact that your stack booted with everything blank *is* the core guarantee. + +**Do this / expect this:** + +```bash +docker-compose -f local.yml run --rm django python manage.py check +# → "System check identified no issues" +``` + +```python +from django.conf import settings +from sde_collections.utils.aws import get_boto3_session +s = get_boto3_session() +print(s.region_name) # → us-east-1 +print(bool(settings.SDE_AWS_ACCESS_KEY_ID)) # → False — default-chain branch taken +``` + +**Phase tests** — expect `4 passed`: + +```bash +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_aws_utils.py -v +``` + +--- + +## Step 1 — The status vocabulary (Phase 1) + +**Where we are.** The workflow diagram above needs six statuses that didn't exist before: +`Scraping Successful` (21), `Test Indexing` (22), `Scraping Failed` (23), `Indexing Failed on +Test` (24), `Indexing Failed on Prod` (25), `Production Indexing` (26). Phase 1 added them, +fixed both colour maps to never `KeyError` on an unmapped status, and made failure statuses +render **red** instead of neutral. + +**Do this** — at `http://localhost:8001/`, find *Aurorasaurus — Reporting Auroras from the Ground +Up* in the list (search box helps): + +1. Open its **workflow status dropdown** — all 26 statuses are listed; the filter panel shows + them too. +2. Open its detail page → **Workflow History** tab. Every transition the walkthrough makes from + here on will appear in this tab — check back after each step. + +**Expect this:** dropdown renders with no JS console errors; failure statuses show as red +buttons (`btn-danger`). + +**Spot-check in shell:** + +```python +from sde_collections.models.collection_choice_fields import WorkflowStatusChoices +from sde_collections.models.collection import Collection +print(len(WorkflowStatusChoices.choices)) # → 26 +c = Collection(workflow_status=WorkflowStatusChoices.SCRAPING_FAILED) +print(c.workflow_status_button_color) # → btn-danger +``` + +**Phase tests** — expect `74 passed` (parametrized over every status × both colour maps; also +covers the Phase 5 trigger table): + +```bash +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_workflow_status_triggers.py -v +``` + +--- + +## Step 2 — Inference is dormant, not dead (Phase 2) + +**Where we are.** The old pipeline routed three hard-coded TDAMM collections through an ML +inference job between scraping and curation. That pipeline is now gated off +(`INFERENCE_ENABLED=False`): `queue_necessary_classifications()` short-circuits straight to the +delta migration for *every* collection. This matters for Step 4 — it's why the ingest reaches +*Ready for Curation* in seconds instead of stranding behind a queued inference job that never +runs. + +**Do this / expect this:** + +```python +from django_celery_beat.models import PeriodicTask +print(list(PeriodicTask.objects.filter(task__startswith="inference").values("name", "enabled"))) +# → both rows enabled=False +``` + +The disable is durable: re-running `manage.py migrate` re-asserts `enabled` from the flag (the +`post_migrate` signal owns these rows — a hand-edit in the admin does not survive a deploy, by +design). Optionally watch `docker-compose -f local.yml logs -f celerybeat` — no inference ticks. + +**Phase tests** — expect `7 passed`: + +```bash +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_inference_flag.py -v +``` + +--- + +## Step 3 — Requesting a scrape (Phase 3) + +**Where we are.** In production, a curator sets **Ready for Engineering** and COSMOS reacts: +`build_job_json()` merges the collection's seed URL with any per-collection +`ScraperConfigOverride` (only non-null fields are emitted — the crawler supplies its own +defaults), and `send_job_to_crawler()` drops the JSON into the crawler's inbox on EC2 via SSM, +recording a `ScrapeDispatch` row (the poller's freshness reference). For Aurorasaurus **this +already happened** — the Aug-13 crawl in the bucket *is* the output of this hop. So locally we +verify the machinery two ways: the job-builder logic in the shell, and the trigger via its +graceful failure path. + +**Do this (1) — job-builder spot-checks:** + +```python +from sde_collections.models.collection import Collection +from sde_collections.models.scraper_config import ScraperConfigOverride +from sde_collections.scraping.job_builder import build_job_json + +c = Collection.objects.get(config_folder="aurorasaurus_reporting_auroras_from_the_ground_up") +print(build_job_json(c)) +# → {"seed": "https://aurorasaurus.org/", "collection_id": "aurorasaurus_reporting_auroras_from_the_ground_up"} + +ScraperConfigOverride.objects.update_or_create(collection=c, defaults={"max_pages": 25, "delay": None}) +print(build_job_json(c)) +# → adds "max_pages": 25; "delay" ABSENT (null overrides are never emitted) +# (max_pages=25 is exactly the override the real Aug-13 crawl ran with — see its summary) + +ScraperConfigOverride.objects.filter(collection=c).update(max_pages=200_000) +try: + build_job_json(c) +except ValueError as e: + print("cap enforced:", e) # → the crawler rejects >100_000, so COSMOS refuses to send it + +ScraperConfigOverride.objects.filter(collection=c).delete() # clean up +``` + +**Do this (2) — the trigger, via its failure path.** In the UI set the collection to +**Ready for Engineering**, then watch: + +```bash +docker-compose -f local.yml logs -f celeryworker +``` + +**Expect this:** within seconds — +`Scrape dispatch failed for aurorasaurus_reporting_auroras_from_the_ground_up: Unable to locate credentials` +and the status flips to **Scraping Failed** (red). That failure *is* the proof: the trigger +fired, the SSM call was attempted, and the error path held (no exception escaped, no +`ScrapeDispatch` row recorded — check Admin → **Scrape dispatches**, a read-only list). With +real crawler settings this same click lands a job JSON in `/opt/sde-crawler/jobs/incoming/`. + +Also verifiable: Admin → **Scraper config overrides** is the curator-facing override editor, and +`manage.py dispatch_scrape --collection ` is the manual CLI re-dispatch (same code path; +locally it exits with a `CommandError` after marking *Scraping Failed* — expected). + +**Phase tests** — expect `10 passed`: + +```bash +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_scrape_dispatch.py -v +``` + +--- + +## Step 4 — Ingesting the real crawl (Phase 4) ★ the centerpiece + +**Where we are.** In production, `poll_scrape_jobs` (a beat task, every 5 min) notices the fresh +completion summary in S3 and enqueues `ingest_scraped_collection`, which: + +1. **claims** the collection with an atomic compare-and-swap + (*Ready for Engineering / Engineering in Progress → Scraping Successful*) so two ingests can + never write concurrently; +2. replaces its `DumpUrl`s with the scraped documents; +3. hands off to the delta migration, which diffs the new dump against the existing `CuratedUrl`s + and promotes the status to **Ready for Curation**. + +We run exactly that path — real S3, real documents, `claim=True` — injecting your short-lived +SSO credentials and the real bucket into a one-shot container (the env file stays untouched; +`SDE_AWS_*` stay blank so the default chain picks up the session token). + +**Do this** — from the repo root, after `aws sso login --profile sde-dev`: + +```bash +eval "$(aws configure export-credentials --profile sde-dev --format env)" +docker-compose -f local.yml run --rm \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -e SDE_S3_BUCKET=sdecrawlerstack-crawlbucket0d63eba8-lhkxqnh8ophy \ + django python manage.py shell -c " +from sde_collections.models.collection import Collection +from sde_collections.models.collection_choice_fields import WorkflowStatusChoices +from sde_collections.tasks import ingest_scraped_collection + +c = Collection.objects.get(config_folder='aurorasaurus_reporting_auroras_from_the_ground_up') +c.workflow_status = WorkflowStatusChoices.ENGINEERING_IN_PROGRESS # claimable; no trigger +c.save() + +print(ingest_scraped_collection(c.id)) # claim=True: the full production path +c.refresh_from_db() +print('status:', c.get_workflow_status_display()) +print('dump_urls:', c.dump_urls.count()) +" +``` + +**Expect this** (observed on the 2026-08-14 run): + +``` +Ingested 25 documents for aurorasaurus_reporting_auroras_from_the_ground_up (replaced 0). +status: Scraping Successful +dump_urls: 25 +``` + +The ingest ends by enqueueing the migration, which the **celeryworker** executes (no S3 needed +there). Give it a few seconds, then: + +```python +from sde_collections.models.collection import Collection +c = Collection.objects.get(config_folder="aurorasaurus_reporting_auroras_from_the_ground_up") +print(c.get_workflow_status_display()) # → Ready for Curation +print(c.dump_urls.count(), c.delta_urls.count(), c.curated_urls.count()) +``` + +**Expect this:** `Ready for Curation`, dumps `0` (cleared by the migration), curated still `25` +— and deltas **`0`**. Zero is the *correct* real-world answer here: the Aug-13 crawl matches the +already-curated set exactly, so the differ found nothing new, changed, or deleted. A re-crawl +with actual site changes would surface exactly those pages as deltas. In the worker log you'll +also see the ingest-summary Slack attempt (caught error with a dummy webhook). + +**Verify in the UI:** the collection shows **Ready for Curation**; **Workflow History** now has +*Engineering in Progress → Scraping Successful → Ready for Curation*. + +Two failure modes worth knowing (both unit-tested): a completed crawl with +`documents_scraped == 0` is marked **Scraping Failed** (an empty crawl must never silently +publish an empty collection), and stale S3 results older than the latest `ScrapeDispatch` are +invisible to the poller (re-dispatch safety). + +**Phase tests** — expect `15 passed`: + +```bash +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_scrape_ingest.py -v +``` + +--- + +## Step 5 — Curation and the hand-off to test indexing (Phase 5) + +**Where we are.** *Ready for Curation → Curated* is the human part of the workflow: curators +review the deltas (exclude URLs, fix titles, tag divisions) in the UI. Setting **Curated** fires +the rewired dispatcher (`handle_workflow_status_change`), which does two things: promotes the +deltas into `CuratedUrl`s, and fires `index_collection_to_test` — the Phase 7 hand-off. With the +indexer's settings blank, that second hop fails gracefully: the wiring proof again. + +**Do this** — in the UI (or shell), walk the collection +*Ready for Curation → Curation in Progress → Curated*. With zero deltas there is nothing to +review this time — which is itself the honest outcome — so this step is mostly about the +trigger. Watch the worker log as you set **Curated**. + +**Expect this:** + +- `CuratedUrl`s: still 25, `DeltaUrl`s: 0 (promote with an empty delta set is a no-op — the + curated data is never touched unnecessarily). +- Worker log: `Index dispatch (test) failed for aurorasaurus_…: SDE_INDEX_BUCKET is not + configured — cannot export`, and the status lands on **Indexing Failed on Test** (red). +- With the `INDEXING_*` settings filled in from the dev stacks, this same click would export + `curated_collections/{cf}/{run_id}/documents.jsonl` + `manifest.json` to S3, `ecs:RunTask` the + WEB_COSMOS indexer, and land on **Test Indexing**. + +**Phase tests:** the trigger table lives in the same suite as Step 1 (`74 passed`); it asserts +`CURATED` promotes *and* enqueues test indexing, `QC_*` enqueues prod indexing, and no Sinequa +method is called on any transition. + +--- + +## Step 6 — Sinequa is gone (Phase 6) + +**Where we are.** Everything the old pipeline used — the Sinequa API client, XML config +generation, the GitHub configs push — was deleted outright in Phase 6. The whole walkthrough you +just did ran without any of it; this step proves the removal is total. + +**Do this / expect this:** + +```bash +ls sde_collections/sinequa_api.py config_generation default_scraper.xml 2>&1 +# → "No such file or directory" for each + +grep -ri "sinequa_api\|XmlEditor\|GitHubHandler\|PyGithub" \ + --include='*.py' --include='*.html' --include='*.js' sde_collections/ config/ +# → empty (a plain grep for "sinequa" still hits comments/help_text — historical, retained +# deliberately, plus the SourceChoices.ONLY_IN_SINEQUA_CONFIGS data value) + +docker-compose -f local.yml run --rm django python manage.py check # boots with NO Sinequa env vars +docker-compose -f local.yml run --rm django pytest sde_collections/tests/ # → 308 passed +``` + +**In the UI:** no Sinequa config link on the collection detail page; no +`import_from_sinequa` / `push_to_github` in `manage.py --help`. + +--- + +## Step 7 — The indexing hand-off, resolved (Phase 7) + +**Where we are.** Phase 7 filled the Phase 5 stubs with the real contract: mint a `run_id`, +export curated docs to `curated_collections/{cf}/{run_id}/` (manifest written **last** = export +complete), assume the cross-repo dispatch role, `ecs:RunTask` the WEB_COSMOS indexer with a +command override, record an `IndexDispatch` row, and let `poll_index_runs` (beat, every 2 min) +resolve the run by reading `index_runs/{cf}/{run_id}/status.json` from S3 — never +`ecs.describe_tasks`. The `run_id` namespacing means an old run's status can never satisfy a +newer dispatch. + +The indexer is live in the dev account, but a local walkthrough has no business starting real +Fargate tasks — so here we simulate **its half only** — the `status.json` responses — while +running COSMOS's poller for real. Three outcomes, continuing from *Indexing Failed on Test*: + +**Do this** — paste into `shell_plus`: + +```python +from unittest import mock +from sde_collections.models.collection import Collection +from sde_collections.models.collection_choice_fields import WorkflowStatusChoices +from sde_collections.models.indexing import IndexDispatch +from sde_collections import tasks + +c = Collection.objects.get(config_folder="aurorasaurus_reporting_auroras_from_the_ground_up") + +# --- Outcome 1: test run SUCCEEDS → status holds at Test Indexing; the indexer-produced +# validation report (count/title QC) is posted to Slack for the curator to judge --- +c.workflow_status = WorkflowStatusChoices.TEST_INDEXING # no trigger on this status +c.save() +IndexDispatch.objects.create(collection=c, run_id="aurora-e2e-1", target="test", + task_arn="arn:demo", previous_workflow_status=WorkflowStatusChoices.CURATED) +validation = {"expected_count": 25, "indexed_count": 25, "count_matches": True, + "title_match_rate": 1.0, "titles_missing_in_index": [], "titles_only_in_index": []} +with mock.patch.object(tasks, "fetch_run_status", return_value={"state": "succeeded"}), \ + mock.patch.object(tasks, "fetch_validation_report", return_value=validation): + print(tasks.poll_index_runs()) # → "Resolved 1 index run(s)." +c.refresh_from_db() +print(c.get_workflow_status_display()) # → Test Indexing (curator now sets QC from the report) +``` + +**Now the QC trigger, for real.** In the UI (or shell) set **QC: Minor Issues** and watch the +worker: `Index dispatch (prod) failed … SDE_INDEX_BUCKET is not configured` → status +**Indexing Failed on Prod**. That's the `QC_* → index_collection_to_prod` trigger, proven the +same way as Step 5. Then resolve the prod run: + +```python +# --- Outcome 2: prod run SUCCEEDS → final status mirrors the QC status it entered with --- +c.refresh_from_db() +c.workflow_status = WorkflowStatusChoices.PRODUCTION_INDEXING +c.save() +IndexDispatch.objects.create(collection=c, run_id="aurora-e2e-2", target="prod", + task_arn="arn:demo", + previous_workflow_status=WorkflowStatusChoices.QUALITY_CHECK_MINOR) +with mock.patch.object(tasks, "fetch_run_status", return_value={"state": "succeeded"}): + print(tasks.poll_index_runs()) +c.refresh_from_db() +print(c.get_workflow_status_display()) # → Prod: Minor Issues ← THE FINISH LINE + +# --- Outcome 3 (optional): an UNKNOWN state is a failure, never a success --- +c.workflow_status = WorkflowStatusChoices.TEST_INDEXING +c.save() +IndexDispatch.objects.create(collection=c, run_id="aurora-e2e-3", target="test", + task_arn="arn:demo", previous_workflow_status=WorkflowStatusChoices.CURATED) +with mock.patch.object(tasks, "fetch_run_status", + return_value={"state": "needs_confirmation", "error": None}): + print(tasks.poll_index_runs()) +c.refresh_from_db() +print(c.get_workflow_status_display()) # → Indexing Failed on Test +``` + +**Expect this** (all three observed on the 2026-08-14 run): `Test Indexing` held with the +validation report posted (caught Slack error on a dummy webhook); `Prod: Minor Issues` as the +finish line; `Indexing Failed on Test` for the unknown state. In Admin → **Index dispatches** +the `aurora-e2e-*` rows are read-only with `completed_at` stamped. + +**Phase tests** — expect `19 passed`: + +```bash +docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_indexing_dispatch.py -v +``` + +--- + +## The finish line — what you just proved + +Open the collection's **Workflow History** tab. The full journey reads: + +> … → Ready for Engineering → **Scraping Failed** *(Step 3: dispatch wiring, graceful failure)* +> → Engineering in Progress → **Scraping Successful** *(Step 4: real S3 ingest, CAS claim)* +> → **Ready for Curation** *(Step 4: migration + zero-delta diff)* +> → Curation in Progress → **Curated** *(Step 5: promote + test hand-off)* +> → **Indexing Failed on Test** *(Step 5: graceful failure = trigger proof)* +> → **Test Indexing** *(Step 7: simulated indexer success, validation posted)* +> → **QC: Minor Issues** → **Indexing Failed on Prod** *(Step 7: prod trigger proof)* +> → **Production Indexing** → **Prod: Minor Issues** *(Step 7: QC mirroring — done)* + +Everything COSMOS-side ran for real: settings/credential plumbing (P0), the status vocabulary +(P1), inference-off migration (P2), dispatch triggers and job building (P3), real-S3 ingest with +claim semantics (P4), curation triggers and promote (P5), a Sinequa-free codebase (P6), and the +export/dispatch/poll machinery (P7). The only simulated piece was the indexer's `status.json`. +Its dev stacks are already deployed, so nothing further has to be built: fill a COSMOS +environment's `INDEXING_*` values from those stack outputs and Steps 5 and 7 stop failing +gracefully — this same collection is the natural first candidate for the fully-real closed loop. + +The walkthrough is repeatable end to end: Step 4's ingest replaces dumps and re-diffs, promote +is idempotent, and each `IndexDispatch` gets a fresh `run_id`. + +--- + +## Appendix A — Fully-offline ingest (no dev-AWS access) + +If you can't reach the dev bucket, Step 4 can run with mocked S3 instead. The only S3 read in the +ingest path is `sde_collections/scraping/s3_results.py::_get_object` — patch it with an in-memory +fake and call the task synchronously. + +> **⚠️ Use a scratch collection, never a real curated one.** The migration diffs the fake dump +> against the existing `CuratedUrl`s — on a collection with a real curated set, 5 fake documents +> produce a *deletion-marker delta for every real URL*. Create a throwaway collection in the +> admin (name + seed URL + division; `config_folder` auto-generates) and use its `config_folder` +> below. + +```python +CONFIG_FOLDER = "" # ⚠️ EDIT FIRST + +import io, json +from datetime import datetime, timezone as tz +from unittest import mock +from botocore.exceptions import ClientError + +from sde_collections.models.collection import Collection +from sde_collections.models.collection_choice_fields import WorkflowStatusChoices +from sde_collections.tasks import ingest_scraped_collection + +c = Collection.objects.get(config_folder=CONFIG_FOLDER) +c.workflow_status = WorkflowStatusChoices.ENGINEERING_IN_PROGRESS +c.save() + +# 5 fake documents in the crawler's exact 7-field shape; URLs must be globally unique +DOCS = [ + {"url": f"https://{c.config_folder}.demo.local/page{i}", + "title": f"Demo Page {i}", "full_text": f"Body text for demo page {i}.", + "content_type": "text/html", "seed": "https://demo.local", + "host": "demo.local", "depth": 1} + for i in range(1, 6) +] +SUMMARY = {"collection_id": c.config_folder, "documents_scraped": len(DOCS)} + +def fake_get_object(key): + payload = { + f"scraped_collections/{c.config_folder}.json": DOCS, + f"failure_logs/{c.config_folder}_failures_summary.json": SUMMARY, + }.get(key) + if payload is None: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(json.dumps(payload).encode()), + "LastModified": datetime.now(tz.utc)} + +with mock.patch("sde_collections.scraping.s3_results._get_object", side_effect=fake_get_object): + print(ingest_scraped_collection(c.id)) # → "Ingested 5 documents for …" + +c.refresh_from_db() +print(c.get_workflow_status_display()) # → Scraping Successful; worker then → Ready for Curation +print(c.dump_urls.count()) # → 5 +``` + +From there the walkthrough continues identically at Step 5 (you'll have 5 fake deltas to +"curate" instead of zero). The zero-document failure case: rerun with +`SUMMARY = {..., "documents_scraped": 0}` (status reset to *Engineering in Progress* first) → +**Scraping Failed**. Idempotency: rerun with `ingest_scraped_collection(c.id, claim=False)` — +twice yields 5 DumpUrls, not 10 (this is also what +`manage.py ingest_scrape_results --collection ` calls; that command always reads real S3, so +locally it fails on the blank bucket by design). + +--- + +## Appendix B — What this guide deliberately does NOT verify + +These need real AWS beyond the crawler bucket. Nothing here is blocked on infrastructure — the +crawler stack and the indexer's dev stacks both exist — they are simply out of scope for a local +run, and are checked by hand against dev AWS instead: + +- A real SSM `send-command` reaching the crawler inbox on `i-0b6a61d95888886f4` (the Step 3 + success path — its output for Aurorasaurus already exists, which is what Step 4 consumed). +- The real export → `sts:AssumeRole` → `ecs:RunTask` → `status.json` loop against + `sde-cosmos-indexing-dev` (Step 5/7 success paths), which runs from a COSMOS host whose instance + role may assume `CosmosIndexingDispatchRole-dev`. +- Slack delivery, if you ran with a dummy webhook. +- The stall-timeout paths in real time (both unit-tested; live simulation means waiting + `SCRAPE_STALL_TIMEOUT_HOURS` / `INDEX_STALL_TIMEOUT_HOURS`). +- The beat pollers firing on schedule (`SCRAPE_POLL_ENABLED` / `INDEX_POLL_ENABLED=True`) — with + blank buckets they would only log S3 errors; their rows' existence and flag-gating are + verified in Steps 2/4/7. diff --git a/README.md b/README.md index ab3da78b..408de230 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,49 @@ COSMOS is a web application designed to manage collections indexed in NASA's Science Discovery Engine (SDE), facilitating precise content selection and allowing metadata modification before indexing. +## How the pipeline works + +A collection moves through scrape → curate → index. COSMOS orchestrates that flow but does not +crawl or index anything itself: it hands work to two external systems and watches S3 for results. +Every hop is a file in S3 plus a poller — there are no callbacks. + +``` + COSMOS (Django + Celery) crawl4ai host sde-api-scrapers (ECS Fargate) + ──────────────────────── ─────────────── ────────────────────────────── + Ready for Engineering ──SSM──▶ crawl ──▶ S3 + Scraping Successful ◀──S3─── poll_scrape_jobs (5 min) + Ready for Curation → (curator works in the UI) + Curated ──S3 export + ecs:RunTask──────────────────────────▶ index into OpenSearch + Test Indexing ◀──S3─── poll_index_runs (2 min) ◀──────────── status.json / validation.json + QC: Perfect / QC: Minor Issues ──▶ prod run ──▶ Prod: Perfect / Prod: Minor Issues +``` + +The curator drives this by changing `workflow_status` in the admin; a `post_save` handler +(`sde_collections/models/collection.py::handle_workflow_status_change`) fires the matching Celery +task. Failures land on **Scraping Failed**, **Indexing Failed on Test**, or **Indexing Failed on +Prod** rather than silently stalling. + +COSMOS holds no OpenSearch or SageMaker credentials. Chunking, vectorizing, indexing, and the QC +validation report all happen in the indexer repo (`sde-api-scrapers`). COSMOS only writes exports +to S3, assumes one IAM role, and calls `ecs:RunTask`. + +### Where things are documented + +| Doc | What it covers | +|---|---| +| [`WORKFLOW.md`](./WORKFLOW.md) | The curation workflow end to end, with a diagram | +| [`sde_collections/DEPLOYMENT.md`](./sde_collections/DEPLOYMENT.md) | How deploys actually work today, and the CI/CD gaps | +| [`LOCAL_VERIFICATION_GUIDE.md`](./LOCAL_VERIFICATION_GUIDE.md) | Verifying the pipeline locally, phase by phase | +| [`sde_collections/models/README_STATUS_TRIGGERS.md`](./sde_collections/models/README_STATUS_TRIGGERS.md) | What each status change triggers | +| [`sde_collections/models/README_LIFECYCLE.md`](./sde_collections/models/README_LIFECYCLE.md) | Dump → Delta → Curated URL lifecycle | + +> **Note on configuration:** every setting that gates a dispatch (S3 buckets, ECS cluster, task +> family, IAM role ARN, subnets, security groups, and both `*_POLL_ENABLED` flags) defaults to +> blank or off in `config/settings/base.py`. Nothing dispatches until a host is explicitly wired. +> The pollers are `django_celery_beat` database rows written by a `post_migrate` receiver +> (`sde_collections/signals.py`), so enabling a poller requires `manage.py migrate` — restarting +> the services alone will not do it. + ## Basic Commands ### Building the Project @@ -202,20 +245,22 @@ $ pip install celery ### Running a Celery Worker +Run these from the **repository root** — the folder containing `manage.py` and `config/`. For +Celery's import magic to work, the working directory matters. + ```bash -$ cd sde_indexing_helper $ celery -A config.celery_app worker -l info ``` -Please note: For Celery's import magic to work, it is important where the celery commands are run. If you are in the same folder with manage.py, you should be right. - ### Running Celery Beat Scheduler ```bash -$ cd sde_indexing_helper $ celery -A config.celery_app beat ``` +Note that beat schedules in this project are `django_celery_beat` **database rows**, not a +`CELERY_BEAT_SCHEDULE` setting — see the configuration note at the top of this file. + ### Pre-Commit Hook Instructions To install pre-commit hooks: @@ -233,21 +278,40 @@ Sign up for a free account at [Sentry](https://sentry.io/signup/?code=cookiecutt ## Deployment -Refer to the detailed [Cookiecutter Django Docker documentation](http://cookiecutter-django.readthedocs.io/en/latest/deployment-with-docker.html). - -## Importing Candidate URLs from the Test Server +See [`sde_collections/DEPLOYMENT.md`](./sde_collections/DEPLOYMENT.md) for how deploys work on this +project — the compose stacks, the branch flow (`dev` → `staging` → `production`), the hand-maintained +per-host env files, and the current CI/CD gaps. Deployment is manual today: SSH to the host and +rebuild in place. -Documented [here](https://github.com/NASA-IMPACT/sde-indexing-helper/wiki/How-to-bring-in-Candidate-URLs-from-the-test-server). +Background on the underlying Docker setup is in the +[Cookiecutter Django Docker documentation](http://cookiecutter-django.readthedocs.io/en/latest/deployment-with-docker.html). ## Adding New Features/Fixes We welcome contributions to improve the project! Before you begin, please take a moment to review our [Contributing Guidelines](./CONTRIBUTING.md). These guidelines will help you understand the process for submitting new features, bug fixes, and other improvements. -## Job Creation +## Dispatching a Scrape + +Scrapes are normally triggered by setting a collection's `workflow_status` to **Ready for +Engineering** in the admin, which dispatches a job to the crawl4ai host over AWS SSM. + +To dispatch one by hand: + +```shell +docker-compose -f local.yml run --rm django python manage.py dispatch_scrape --collection +``` + +Results are picked up automatically by the `poll_scrape_jobs` beat task (every 5 minutes, gated by +`SCRAPE_POLL_ENABLED`). To ingest a completed scrape manually instead: + +```shell +docker-compose -f local.yml run --rm django python manage.py ingest_scrape_results --collection +``` -Eventually, job creation will be done seamlessly by the webapp. Until then, edit the `config.py` file with the details of what sources you want to create jobs for, then run `generate_jobs.py`. +## Code Structure -## Code Structure for SDE_INDEXING_HELPER +The Django project package is still named `sde_indexing_helper/` (COSMOS is the product name; the +package was not renamed). - Frontend pages: - HTML: `/sde_indexing_helper/templates/` @@ -255,6 +319,17 @@ Eventually, job creation will be done seamlessly by the webapp. Until then, edit - CSS: `/sde_indexing_helper/static/css` - Images: `/sde_indexing_helper/static/images` +- Pipeline code, all under `sde_collections/`: + - `scraping/` — dispatching crawls and reading their results: + `job_builder.py` (job JSON), `ssm_dispatch.py` (SSM send-command to the crawl4ai host), + `s3_results.py` (freshness checks and result fetch) + - `indexing/` — the hand-off to the indexer: + `export.py` (writes `documents.jsonl`, then `manifest.json` last), + `dispatch.py` (`sts:AssumeRole` → `ecs:RunTask`), `run_status.py` (reads `status.json`) + - `tasks.py` — the Celery tasks, including both pollers + - `signals.py` — the `post_migrate` receiver that writes the beat schedule rows + - `models/` — collections, URL lifecycle, and patterns (see the `README_*.md` files there) + ## Running Long Scripts on the Server ```shell diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1ced2857..f6943964 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,4 +1,28 @@ # COSMOS Release Notes +## Unreleased — `cosmos-rewiring` (Sinequa removal, crawl4ai + web-indexing hand-off) + +Sinequa, the GitHub config push and XML generation are removed. Collections are now scraped by the +crawl4ai crawler (dispatched over SSM, results ingested from S3) and indexed by the `sde-api-scrapers` +`WEB_COSMOS` ECS task (curated export to S3, `ecs:RunTask`, status polled from S3). Six workflow +statuses are added (Scraping Successful/Failed, Test Indexing, Indexing Failed on Test/Prod, +Production Indexing). See `WORKFLOW.md` and `sde_collections/DEPLOYMENT.md`. + +### Behaviour changes to be aware of +- **TDAMM classification threshold now takes effect.** `map_classification_to_tdamm_tags` previously + accepted `threshold` and ignored it; it is now honoured (`inference/utils/classification_utils.py`). + Inference is shipped disabled (`INFERENCE_ENABLED=False`); when re-enabled, collections will receive + different TDAMM tags than earlier runs at the same `TDAMM_CLASSIFICATION_THRESHOLD`. +- **Orphaned workflow statuses.** `Secret Deployment Started` (8), `Ready for LRM Quality Check` (10), + `Merge Pending` (17) and `Indexing Finished on Dev` (20) remain selectable but nothing advances them + any more (the Sinequa/LRM steps that consumed them are gone). Collections currently parked in those + statuses need to be moved by hand — audit them before/after deploying. +- Celery beat schedules for the two S3 pollers are DB rows written on `post_migrate`; their `enabled` + flag is re-asserted from `SCRAPE_POLL_ENABLED` / `INDEX_POLL_ENABLED` on **every** `migrate`. + Toggling a poller therefore requires `manage.py migrate`, not just a restart. +- Status-triggered Celery tasks are now enqueued with `transaction.on_commit`. +- A failed re-scrape (`Re-Indexing Needed`) no longer rewrites the collection's live workflow status; + it clears the reindexing request and posts a Slack alert instead. + ## v3.0.0 from v2.0.1 COSMOS v3.0.0 introduces several major architectural changes that fundamentally enhance the system's capabilities. The primary feature is a new website reindexing system that allows COSMOS to stay up-to-date with source website changes, addressing a key limitation of previous versions where websites could only be scraped once. This release includes comprehensive updates to the data models, frontend interface, rule creation system, and backend processing along with some bugfixes from v2.0.1. diff --git a/REVIEW_AND_DEPLOYMENT_PREP.md b/REVIEW_AND_DEPLOYMENT_PREP.md new file mode 100644 index 00000000..63dedeb3 --- /dev/null +++ b/REVIEW_AND_DEPLOYMENT_PREP.md @@ -0,0 +1,391 @@ +# Review & Deployment Prep — `cosmos-rewiring` + `sde-api-scrapers` web indexing + +Prepared 2026-08-25 against COSMOS `cosmos-rewiring` @ `c1505d36`; review fixes committed the same day as +`90dbad86` ("bug fixes") — branch is now 16 commits over `dev`, 137 files, +7327/−9375, tree clean. +`sde-api-scrapers` `web-indexing` @ `8dde345` (local checkout; see §1 — the work itself lives on `develop`). + +--- + +## 0. TL;DR + +1. **COSMOS branch review findings are fixed and committed (`90dbad86`, 2026-08-25)** — §4 has a + status column. Fixed: A (`on_commit`), B/C (dead Slack transitions), D (re-scrape clobbering prod status), + E (unwired-host guard), G (in-crawl URL de-dup), H, I, K, template leftovers, gitleaks config, release notes, + and the test gaps. Still open: F (whole-crawl-in-memory ingest), L (clock skew), cross-collection duplicate + URLs (design call), DB indexes, and the out-of-band credential rotation. Full `init.sh` suite is green. +2. **`sde-api-scrapers/web-indexing` is already merged and deployed.** PR #47 (`c2aebad`) landed on `develop` + and the dev stacks deployed 2026-08-20. The local `web-indexing` branch is now *behind* `develop` — diffing it + against `develop` yields a **revert of the RDR prod-freeze work (PR #49/#50)**. Do not open a PR from it; + rebase or delete it. "Review the indexer" means reviewing `web/` + the shared uploader changes as they sit on + `develop`. +3. **One gate remains before the closed loop can run: C.2** — wire the P7 env block on the **staging** host, + run `migrate` (the beat rows come from `post_migrate`), recreate containers. Infra on both sides is in place. +4. **One indexer bug must land before any test/prod tier deploy (R1):** the guards read the tier-capped endpoint + but `APIOpenSearchUploader` writes to bare `OPENSEARCH_ENDPOINT`. Harmless in dev (same collection), a + guard-bypass in prod. +5. **The production DB password is still in git history** (scrubbed from `SQLDumpRestoration.md` on this branch, + but present on `dev`). Rotate it; the scrub alone does nothing. The gitleaks hook that should have caught it + pointed at a missing `gitleaks-config.toml` — that file now exists and the hook passes, but history is + history. +6. COSMOS tree is clean. The scrapers repo still has a status-update edit to the task tracker and an + **untracked `COSMOS_INDEX_REAL_RUN.md`** which is the best description of the hand-off contract anywhere — + commit it. + +--- + +## 1. Branch state + +| Repo | Branch | vs base | Notes | +|---|---|---|---| +| COSMOS | `cosmos-rewiring` @ `90dbad86` | 16 commits over `dev` (Phase 0–7 + review fixes) | Tree clean. CI (`run_full_test_suite.yml`) runs only on PRs to `dev`, `paths-ignore: '**/*.md'`. | +| sde-api-scrapers | `web-indexing` @ `8dde345` | `origin/develop..web-indexing` = 1 commit (the merge); `web-indexing..origin/develop` = 8 commits | Web indexing merged via PR #47. Diff vs `develop` = removal of `RDR_API_BASE_URLS`/`_resolve_base_url()`/`schedule_enabled` — i.e. it would re-enable the frozen prod RDR EventBridge rule and reintroduce the silent sandbox fallback. | + +Uncommitted / untracked: +- COSMOS: nothing. (`.pre-commit-config.yaml` pyupgrade bump `v3.20.0 → v3.21.2` went in with `90dbad86`.) +- scrapers: `Web Indexing - Task Plan & Tracking.md` (records NS.2 done, NS.7 closed, OOB.3 verified, ECR digest + `ae1c9e02…` co-tagged `c2aebad…`, image linux/amd64 only); `COSMOS_INDEX_REAL_RUN.md` (untracked runbook, "not + yet executed end to end"). + +--- + +## 2. What the COSMOS branch does + +**Pipeline in one paragraph.** A curator moves a collection to *Ready for Engineering* → `post_save` enqueues +`dispatch_scrape_job`, which drops a crawl4ai job JSON into the crawler EC2 inbox over SSM `AWS-RunShellScript` +and records a `ScrapeDispatch`. `poll_scrape_jobs` (beat, every 5 min) watches +`s3://{SDE_S3_BUCKET}/failure_logs/{cf}_failures_summary.json`; a summary newer than `dispatched_at` triggers +`ingest_scraped_collection`, which claims the collection with a compare-and-swap, replaces `DumpUrl` rows from +`scraped_collections/{cf}.json`, then runs the existing delta migration → *Ready for Curation*. Inference is +present but dormant (`INFERENCE_ENABLED=False`). On *Curated*, `promote_to_curated()` then +`index_collection_to_test` exports `CuratedUrl` rows as JSONL to `s3://{SDE_INDEX_BUCKET}/curated_collections/{cf}/{run_id}/` +(`manifest.json` written last), assumes `INDEXING_DISPATCH_ROLE_ARN`, and `ecs:RunTask`s the indexer with a full +command override. `poll_index_runs` (every 2 min) reads `index_runs/{cf}/{run_id}/status.json`; test success +posts `validation.json` to Slack and holds at *Test Indexing*; a curator sets QC Perfect/Minor → prod dispatch → +*Production Indexing* → `PROD_PERFECT`/`PROD_MINOR`. Sinequa, GitHub config push, XML generation and the +health-check module are deleted outright. + +### New modules + +| Path | Purpose | +|---|---| +| `sde_collections/scraping/job_builder.py` | Pure `build_job_json(collection)` — seed URL + `collection_id` + non-null `ScraperConfigOverride` fields; refuses `max_pages > 100_000`. | +| `sde_collections/scraping/ssm_dispatch.py` | `send_job_to_crawler` — write `.tmp` → `chown` → `mv -f` into `CRAWLER_INBOX_PATH` via SSM; `shlex.quote`d. | +| `sde_collections/scraping/s3_results.py` | `fetch_summary`/`fetch_documents`/`results_ready` — missing key = not ready; `LastModified <= dispatched_at` = stale. | +| `sde_collections/indexing/export.py` | `export_curated_to_s3` — streams non-excluded `CuratedUrl` to JSONL, manifest last with exact `document_count`. | +| `sde_collections/indexing/dispatch.py` | `run_index_task` — settings guard, `sts:AssumeRole`, fresh ECS client, RunTask with full command override, optional `awsvpc` config. | +| `sde_collections/indexing/run_status.py` | `fetch_run_status`/`fetch_validation_report` from S3 only (never `ecs:DescribeTasks`). | +| `sde_collections/models/scraper_config.py` | `ScraperConfigOverride` (1:1, curator-editable), `ScrapeDispatch`. | +| `sde_collections/models/indexing.py` | `IndexDispatch` (`run_id`, `target`, `task_arn`, `previous_workflow_status`, `completed_at`). | +| `sde_collections/signals.py` | `post_migrate` creates/updates two `django_celery_beat` rows and re-asserts `enabled` from settings on every migrate. | +| `sde_collections/utils/aws.py` | `get_boto3_session()` — `SDE_AWS_*` keys if both set, else instance role. Ignores `DJANGO_AWS_*`. | +| `sde_collections/utils/slack_utils.py` | 10 new transition messages + `send_indexing_validation_report`. | +| `sde_collections/tasks.py` | `dispatch_scrape_job`, `poll_scrape_jobs`, `ingest_scraped_collection`, `_dispatch_index_run`, `index_collection_to_{test,prod}`, `poll_index_runs`. | +| `management/commands/{dispatch_scrape,ingest_scrape_results}.py` | Synchronous manual entry points (`--collection `); ingest uses `claim=False`. | + +### New settings (`config/settings/base.py:346-382`; mirrored in `.env_sample`, `.envs/.local/.django`) + +| Setting | Default | Setting | Default | +|---|---|---|---| +| `AWS_REGION` | `us-east-1` | `SDE_INDEX_BUCKET` | `""` | +| `SDE_S3_BUCKET` | `""` | `INDEXING_ECS_CLUSTER` | `""` | +| `CRAWLER_INSTANCE_ID` | `""` | `INDEXING_TASK_FAMILY` | `""` | +| `CRAWLER_INBOX_PATH` | `/opt/sde-crawler/jobs/incoming` | `INDEXING_CONTAINER_NAME` | `WEB_COSMOSContainer` | +| `SCRAPE_POLL_ENABLED` | `False` | `INDEXING_DISPATCH_ROLE_ARN` | `""` | +| `SCRAPE_STALL_TIMEOUT_HOURS` | `24` | `INDEXING_SUBNETS` | `""` | +| `INFERENCE_ENABLED` | `False` | `INDEXING_SECURITY_GROUPS` | `""` | +| `SDE_AWS_ACCESS_KEY_ID` / `_SECRET_ACCESS_KEY` | `""` (local only) | `INDEX_POLL_ENABLED` | `False` | +| | | `INDEX_STALL_TIMEOUT_HOURS` | `6` | + +Removed: `GITHUB_ACCESS_TOKEN`, `SINEQUA_CONFIGS_*`, `XLI_*`, `LRM_*`. Dropped deps: `PyGithub`, `xmltodict`. +`config/settings/test.py` forces `CELERY_BROKER_URL=memory://` (without it tests publish real tasks). + +### Migrations & statuses +- `0078` — `AlterField` for six new `WorkflowStatusChoices` (21 `SCRAPING_SUCCESSFUL`, 22 `TEST_INDEXING`, + 23 `SCRAPING_FAILED`, 24 `INDEXING_FAILED_ON_TEST`, 25 `INDEXING_FAILED_ON_PROD`, 26 `PRODUCTION_INDEXING`). + Also sweeps unrelated `match_pattern_type` default drift on five delta-pattern models. **No data migration — + none needed**; values are additive and `dev` topped out at 20. +- `0079` — `ScraperConfigOverride`, `ScrapeDispatch`. `0080` — `IndexDispatch`. All additive → rollback-safe. +- Orphaned statuses `8`, `10`, `17`, `20` remain in choices but nothing advances them any more (the + `IndexingInstructionsView` that drove 8→10 is gone). Audit prod rows parked there. + +--- + +## 3. What the indexer side does (`sde-api-scrapers` `develop`) + +- Entry: `python3 api_scraper.py --source WEB_COSMOS --collection --run-id [--target test|prod] + [--reconcile] [--web-index NAME] [--allow-id-collision]` (`api_scraper.py:616,695-728`). Exit 0 iff + `status.state == "succeeded"`. Excluded from `--source ALL`. +- `web/web_pipeline.py` orchestrates: load manifest → ensure index → probe scope → id-collision check → + scan existing ids → deletion guard → index/vectorize → tombstone → write `status.json` (+ `validation.json` + when `--target test`). +- **Contract with COSMOS** (`web/cosmos_source.py:4-11`): + ``` + COSMOS writes curated_collections/{collection_key}/{run_id}/documents.jsonl + curated_collections/{collection_key}/{run_id}/manifest.json ← must be LAST + indexer writes index_runs/{collection_key}/{run_id}/status.json + index_runs/{collection_key}/{run_id}/validation.json (test only) + ``` + Manifest requires `collection_key`, `run_id`, `document_count`; also reads `collection_name`, + `document_type`, `division`. Per-doc passthrough is `url`, `title`, `full_text` only — **`tdamm_tag` is + dropped** at `web/web_processor.py:18-25`. Doc id `/SDE/{collection_key}/|{url}`; `version` = sha256[:32] + of `[title, full_text, document_type, division]`. +- Index: working `sde-web-subset` (59 docs, 2 collections, mapping verified); live `sde-web` (431k docs, + **no `version` field yet** — OOB.1). Mapping in `web/index_mappings/sde_web.json`. +- Runtime: Fargate 2 vCPU / 8 GB, family `web_cosmos-scraper-dev`, container `WEB_COSMOSContainer`, + cluster `api-scrapers-cluster-dev`, logs `/ecs/api-scrapers-dev`. No cron, no monitoring alarm, **no + `ENTRYPOINT`** — the baked command is a placeholder that argparse rejects; COSMOS must send the full + command override (it does: `indexing/dispatch.py:47-58`). +- IAM: `CosmosIndexingDispatchRole-dev` trusts only `indexing-helper-role`; `RunTask` limited to the family + + cluster; `PassRole` on the two ECS roles. Bucket policy grants COSMOS put on `curated_collections/*`, get + on `index_runs/*`. No SSM parameters are used on the web path. +- Tier capping (`infrastructure/config/settings.py:330-350`): dev→dev/dev, test→test/test, prod→test/prod. + +--- + +## 4. Code review checklist — COSMOS + +Status column reflects commit `90dbad86` (2026-08-25); full `init.sh` suite green at that commit. + +### High + +| # | Status | Finding | Where | Resolution | +|---|---|---|---|---| +| A | ✅ fixed | `.delay()` fired inside `post_save`, no `transaction.on_commit`. Admin change views are atomic, so `export_curated_to_s3` could run before `promote_to_curated()`'s rows commit → false `INDEXING_FAILED_ON_TEST`. | `sde_collections/models/collection.py` `handle_workflow_status_change` | `_enqueue_on_commit()` wraps all four `.delay()` calls. Test: `test_task_is_enqueued_only_after_commit`. | +| B | ✅ fixed | Statuses set via queryset `.update()` bypassed `post_save`, so the `SCRAPING_SUCCESSFUL` / ingest-side `SCRAPING_FAILED` Slack messages never fired. | `tasks.py` ingest + `_mark_scrape_failed`; `utils/slack_utils.py` | `notify_status_change()` posts the mapped message after every `.update()`-set transition. Test: `test_claim_posts_the_scraping_successful_notification`. | +| C | ✅ fixed | "Live on Public Prod" keyed on `QC_* → PROD_*`, but the flow goes `PRODUCTION_INDEXING → PROD_*`. | `utils/slack_utils.py` `STATUS_CHANGE_NOTIFICATIONS` | Added `(PRODUCTION_INDEXING, PROD_PERFECT/PROD_MINOR)` pairs. Test: `test_prod_handoff_transitions_are_mapped`. | +| D | ✅ fixed | Ingest `except` unconditionally set `SCRAPING_FAILED`, including on the re-scrape path where the collection may be `PROD_PERFECT`. | `tasks.py` `_mark_scrape_failed` | Only rewrites `workflow_status` when it is in `SCRAPE_FLOW_STATUSES`; otherwise leaves it, resets `reindexing_status` → *Not Needed* (stops the 5-min re-enqueue loop) and posts a Slack alert. Tests: `test_rescrape_ingest_failure_leaves_workflow_status_alone`, `test_rescrape_dispatch_failure_leaves_prod_status_alone`. | +| E | ✅ fixed | Blank `CRAWLER_INSTANCE_ID` → `InstanceIds=[""]` → every *Ready for Engineering* transition on an unwired host failed + @-mention. | `scraping/ssm_dispatch.py` | Settings guard for `CRAWLER_INSTANCE_ID` / `CRAWLER_INBOX_PATH`, mirroring `indexing/dispatch.py`. Test: `test_unconfigured_crawler_refuses_to_dispatch`. | +| F | ❌ open | Whole crawl JSON (`full_text` for up to 100k pages) loaded with one `json.loads`; list held through batching. Worker OOM risk on large collections. | `scraping/s3_results.py` `fetch_documents`; `tasks.py` ingest | Needs a crawler-side JSONL contract or `ijson` (not in requirements). Not changed; size ceiling is `MAX_PAGES_CAP` = 100k. | +| G | ✅ partial | `BaseUrl.url` is `unique=True` **globally**. A duplicate URL in one crawl output — or a URL already in another collection's `DumpUrl` rows — raised `IntegrityError` → whole ingest failed. | `models/delta_url.py:72`; `tasks.py` `_dedupe_by_url` | In-crawl duplicates/blank URLs are now dropped (first wins, logged). **Cross-collection** duplicates still fail — skip-vs-fail is a design decision, see §10. Test: `test_duplicate_urls_in_crawl_output_are_dropped`. | + +### Medium + +| # | Status | Finding | Where | Resolution | +|---|---|---|---|---| +| H | ✅ fixed | Prod mirror map only had `QC_MINOR`; anything else silently became `PROD_PERFECT`. | `tasks.py` `PROD_STATUS_FOR_QC_STATUS` | Explicit two-key map; a prod run entered from a non-QC status holds at `PRODUCTION_INDEXING`, resolves the dispatch, and Slacks "set by hand". Test: `test_prod_success_from_non_qc_status_holds_for_manual_resolution`. | +| I | ✅ fixed | `print()` instead of `logging`; persistent S3 errors were invisible. | `tasks.py`, `utils/slack_utils.py` | Module loggers; `logger.exception` on failure paths so tracebacks are kept. | +| J | ⏸ by design | Beat `enabled` re-asserted on every `post_migrate`: flag flips need `migrate`, admin toggles are reverted on deploy. | `sde_collections/signals.py`; same in `inference/signals.py` | Kept (flag is the source of truth). Now in `RELEASE_NOTES.md` and pinned by `test_signals.py::test_flag_is_reasserted_on_every_migrate`. | +| K | ✅ fixed | `assignPublicIp: ENABLED` hardcoded; SGs only read inside the subnets guard. | `indexing/dispatch.py`; `config/settings/base.py` | New `INDEXING_ASSIGN_PUBLIC_IP` (default `True`; in `.env_sample`). Subnets/SGs read independently; one-without-the-other raises `ValueError`. 3 new tests. | +| L | ❌ open | `results_ready` compares S3 `LastModified` to the Django host clock — skew makes a fresh summary look stale forever. | `scraping/s3_results.py` `results_ready` | A tolerance window would let a quick re-dispatch accept the previous run's summary. Proper fix: snapshot the prior summary's `LastModified` on `ScrapeDispatch` at dispatch time (needs a migration). Decide in review. | +| M | ✅ test / ⏸ docs | Real account ID in a test; instance IDs, IPs, ARNs throughout the planning docs. | `tests/test_indexing_dispatch.py` | Test uses `123456789012`. Planning docs (and this file) are tracked in `90dbad86`; owner decides repo vs wiki before the PR. | +| N | ✅ hook / ⏸ rotate | Prod DB password + RDS host in git history (`SQLDumpRestoration.md` on `dev`); gitleaks hook pointed at a missing config. | `.pre-commit-config.yaml`; `gitleaks-config.toml` | `gitleaks-config.toml` created (extends default rules); `pre-commit run gitleaks --all-files` passes. **Credential rotation still required** — out of band. | + +### Cleanup / leftovers + +| Status | Item | +|---|---| +| ✅ | `collection_detail.html` "View on prod / secret prod" buttons (deleted properties) removed; `utils/generate_deployment_message.py` (unused) trimmed for the same reason. | +| ✅ | `IndexDispatch.TARGET_TEST` / `TARGET_PROD` constants replace bare `"test"`/`"prod"` in `tasks.py`. | +| ✅ | `threshold` behaviour change and orphaned statuses 8/10/17/20 documented in `RELEASE_NOTES.md` ("Unreleased") + `CHANGELOG.md`. | +| ❌ skipped | Cosmetic Sinequa `help_text` strings (`delta_url.py`, `candidate_url.py`, `pattern.py`, `delta_patterns.py`) and `ConnectorChoices.ONLY_IN_SINEQUA_CONFIGS` — Django tracks `help_text`, so editing them generates a migration; not worth the noise on this branch. Legacy `scripts/` and `jupyter_notebooks/` import nothing deleted. | +| ❌ later | Indexes on `IndexDispatch.run_id` / `ScrapeDispatch.collection` — needs a migration; fine at current scale. | +| ➖ non-issue | 5-min `poll_scrape_jobs` vs 10-min ingest `soft_time_limit`: the CAS claim runs *before* `fetch_documents`, so a second invocation exits without downloading. Earlier note was overstated. | + +### Tests +- Branch added: `test_scrape_dispatch.py`, `test_scrape_ingest.py`, `test_indexing_dispatch.py`, + `test_inference_flag.py`, `test_aws_utils.py`. Removed: `test_sinequa_api.py`, `test_import_fulltexts.py`, + `sde_collections/tests.py`, `config_generation/tests/*`. +- Real fixes riding along: `environmental_justice/tests/conftest.py` no longer leaks `ROOT_URLCONF` into later + modules; `test_promote_collection.py` patterns changed from `.*docs.*` to `*docs*` (the old regex idiom matched + nothing); `test_migrate_dump.py` imports the real `DELTA_COMPARISON_FIELDS`. +- Review fixes added: `test_signals.py` (6), `test_management_commands.py` (6), plus on_commit ordering, + Slack-content, re-scrape failure, URL de-dup, unmapped prod mirror, SSM guard and network-config tests in the + existing suites. Trigger tests now use `captureOnCommitCallbacks(execute=True)`. +- Gaps closed: `signals.py`, both management commands, Slack message content, on_commit ordering. Still untested: + F (memory) and L (clock skew) by nature. +- Run: `docker-compose -f local.yml run --rm django bash ./init.sh` then `… coverage report` + (`init.sh` runs each `test_*.py` as its own process). Last run 2026-08-25: all files pass; + `makemigrations --check` clean. + +--- + +## 5. Code review checklist — indexer (`sde-api-scrapers`) + +| # | Finding | Where | Action | +|---|---|---|---| +| R0 | Local `web-indexing` is behind `develop`; a PR from it reverts PR #49/#50 (RDR prod freeze + sandbox guard). | `scrapers/rdr_scraper.py`, `infrastructure/config/settings.py`, `infrastructure/scheduling/scheduling_stack.py:112` | Rebase onto `origin/develop` or delete the branch. | +| R1 | **Split-brain endpoints.** `WebPipeline._resolve_endpoint` tier-caps `self.endpoint` for `ensure_index`/`probe_scope`/collision check/state scan/validate, but `APIOpenSearchUploader` never receives it — six bare `get_opensearch_client()` calls fall back to env `OPENSEARCH_ENDPOINT`. Masked in dev/test (same collection); on prod `--target test` guards TEST and writes PROD. Uploader is a `MagicMock` in `test_web_pipeline.py`, so untested. | `web/web_pipeline.py:110-116`; `uploader/api_opensearch_upload.py:92,187,245,364,454,521` | Thread `endpoint` into the uploader constructor. **Block prod deploy on this.** | +| R2 | `export_incomplete` / skipped deletions set `out["error"]` but `state` is still overwritten to `"succeeded"`; COSMOS branches on `state` only → truncated export indexes partially and reports success. | `web/web_pipeline.py:155,226-228` | Decide: `failed`, or a distinct `succeeded_with_warnings` COSMOS handles explicitly. | +| R3 | Deletion thresholds (`0.90`, `0.25`, `5000`) have silent in-code defaults; only the deployed task def injects them. | `web/deletion_guard.py:28-48` | Acceptable with the drift test; note for local runs. | +| R4 | Hardcoded account `998871305517`, role ARN, three AOSS hostnames; `deploy.yml:163` verify step bakes the account into the S3 name. TEST/PROD COSMOS account IDs commented out (silently no grant, not a synth error). | `infrastructure/config/settings.py:241-243,290-292,310` | Fine for dev; must be resolved for test/prod (NS.5). | +| R5 | Operator-facing error strings tell you to read `FINDING_id_scheme_collision.md §9` — deleted in `8c34fea`. Nine dangling doc refs total. | `web/id_collision.py:15,42,146,172`; `web/scope.py:18`; `web/validate.py:2`; `README.md:99` | Restore §9 or rewrite the strings. | +| R6 | 369 unit tests pass; **zero E2E** (E2E.2–E2E.10 unchecked). `test_infra_web_env.py:374` is misnamed (asserts endpoint divergence, not bucket passthrough). | tracker `:446-459` | C.4 below is the E2E. | +| R8 | 12 id-collision collections (`gcn_circulars`, `CODE_NASA_API`, …) excluded by verbal agreement only; nothing enforces it on either side beyond the runtime refusal. The 0.90 ratio guard does not back up the scope filter; `WEB_DELETION_ABORT_MAX` is the real protection. `AWSV4SignerAuth` rotation fix unverified on a >6 h run. | tracker `:460-477` | Consider a COSMOS-side denylist or an `ScraperConfigOverride` flag. | +| R9 | `documents.jsonl` read 2× per run (3× with `--reconcile`); ids and titles held in memory — the reason for 8 GB. | `web/web_pipeline.py:204-206,267,284` | Note; not a blocker. | +| R10 | Three dependency sources (`requirements.txt`, `pyproject.toml`+`uv.lock`, `infrastructure/requirements.txt`); `requests-aws4auth` still pinned though unused. | | Cleanup. | +| — | `infrastructure/DEPLOYMENT.md` predates the branch; its `run-task` example has no command override and will fail for `WEB_COSMOS`. `API_SCRAPERS_ARCHITECTURE_REVIEW.md` is stale (3 sources, `us-west-1`). | | Update or mark stale. | + +Deploy path: `.github/workflows/deploy.yml` on push to `develop`/`test`/`main` → `pytest tests/` → ECR +`:sha`+`:latest` → `cdk deploy --all` → verify. OIDC trust is branch-pinned, so `workflow_dispatch` from a feature +branch fails at credential setup; rollback is `git revert -m 1 ` + push. + +--- + +## 6. Pre-merge fix list + +**COSMOS (before PR `cosmos-rewiring → dev`)** — committed as `90dbad86` on 2026-08-25; full suite green via `init.sh` +- [x] A — `_enqueue_on_commit` wraps the four `.delay()` calls (`collection.py`) +- [x] B + C — `notify_status_change` posts for `.update()`-set statuses; `PRODUCTION_INDEXING → PROD_*` pairs added +- [x] D — `_mark_scrape_failed` leaves the live `workflow_status` alone on the re-scrape path (clears the reindexing request + Slack alert so the poller doesn't loop) +- [x] E — settings guard in `send_job_to_crawler` +- [x] Template leftovers removed from `collection_detail.html` (and `generate_deployment_message.py`) +- [x] Placeholder account ID in `test_indexing_dispatch.py` +- [x] `gitleaks-config.toml` created (extends defaults) — `pre-commit run gitleaks --all-files` passes +- [x] Commit `.pre-commit-config.yaml` (in `90dbad86`) +- [ ] Decide whether `INDEXING_HANDOFF_TODO.md`, `IMPLEMENTATION_PLAN.md`, `LOCAL_VERIFICATION_GUIDE.md` — and this file — (instance IDs, IPs, ARNs) belong in a public repo, or move to the wiki. All four are tracked as of `90dbad86`; removing them later still leaves them in history. +- [x] Release notes (`RELEASE_NOTES.md` "Unreleased") + `CHANGELOG.md` entry +- [x] G (in-crawl URL de-dup), H (explicit `PROD_STATUS_FOR_QC_STATUS`; unknown entry status holds at `PRODUCTION_INDEXING` + Slack), I (`logging` in `tasks.py`/`slack_utils.py`), tests for `signals.py` and both mgmt commands +- [x] K — `INDEXING_ASSIGN_PUBLIC_IP` setting; subnets/SGs validated together (`indexing/dispatch.py`, `base.py`, `.env_sample`) +- [x] `IndexDispatch.TARGET_TEST/TARGET_PROD` constants used in `tasks.py` +- [ ] F (streaming ingest) — not done; needs a crawler-side JSONL contract or `ijson` +- [ ] L (clock skew in `results_ready`) — not done; see §4 for the migration-based fix +- [ ] G, cross-collection duplicate URLs — design decision (skip vs fail), see §10 +- [ ] Indexes on `IndexDispatch.run_id` / `ScrapeDispatch.collection` — later, needs a migration +- Out of band: **rotate the prod DB credential** + +**Indexer** +- [ ] R0 — rebase/delete local `web-indexing` +- [ ] Commit `COSMOS_INDEX_REAL_RUN.md` and the tracker update +- [ ] R1 — endpoint into `APIOpenSearchUploader` (before any test/prod tier deploy) +- [ ] R2 — settle `export_incomplete` semantics with COSMOS +- [ ] R5 — restore/rewrite the collision remediation text + +--- + +## 7. Deployment runbook (dev, staging host only) + +**Standing rule:** all COSMOS work runs on **staging `i-08f9b2175b70fa05c`** (`ssh staging_cosmos`, +`ec2-user@18.215.146.207`, key `~/.ssh/sde-indexing-helper-staging.pem`). Never the production box +`i-02b3d3e1ac0671952`. `i-0178c998e868792d7` (`COSMOS_Staging_Refresh`) has no instance profile and cannot +dispatch. The COSMOS boxes are not SSM-registered — use SSH. Note `indexing-helper-role` is shared by staging and +production, so the C.1 `sts:AssumeRole` grant already exists on production's role too. + +### Prerequisites — all in place (account `998871305517`, `us-east-1`, profile `sde-dev`) + +| Resource | Value | Status | +|---|---|---| +| Indexing bucket | `sde-cosmos-indexing-dev` | ✅ NS.2 | +| Dispatch role | `arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev` | ✅ NS.2 | +| COSMOS role grant | inline `CosmosIndexingDispatch-dev` on `indexing-helper-role` | ✅ C.1 / NS.6 done 2026-08-20 (the scrapers tracker row is stale) | +| ECS | cluster `api-scrapers-cluster-dev`, family `web_cosmos-scraper-dev:1`, container `WEB_COSMOSContainer` | ✅ | +| ECR | `…/api-scrapers-dev:latest` = `c2aebad` | ✅ | +| Network | VPC `vpc-0265394c8c285afba`, SG `sg-01817cfe4f3629986`, 6 public subnets (below) | ✅ NS.3 values known | +| Index | `sde-web-subset` (59 docs; mapping verified) | ✅ OOB.3 | +| Crawler | `i-0b6a61d95888886f4`, bucket `sdecrawlerstack-crawlbucket0d63eba8-lhkxqnh8ophy`, inbox `/opt/sde-crawler/jobs/incoming` | ✅ | + +### C.2 — wire the staging host + +1. `ssh staging_cosmos`, `cd` to the checkout, `git fetch && git checkout cosmos-rewiring` (or the merged `dev`). +2. Append to `.envs/.production/.django` (from `INDEXING_HANDOFF_TODO.md:293-299`): + ``` + AWS_REGION=us-east-1 + SDE_S3_BUCKET=sdecrawlerstack-crawlbucket0d63eba8-lhkxqnh8ophy + CRAWLER_INSTANCE_ID=i-0b6a61d95888886f4 + CRAWLER_INBOX_PATH=/opt/sde-crawler/jobs/incoming + SCRAPE_POLL_ENABLED=true + INFERENCE_ENABLED=False + + SDE_INDEX_BUCKET=sde-cosmos-indexing-dev + INDEXING_ECS_CLUSTER=api-scrapers-cluster-dev + INDEXING_TASK_FAMILY=web_cosmos-scraper-dev + INDEXING_CONTAINER_NAME=WEB_COSMOSContainer + INDEXING_DISPATCH_ROLE_ARN=arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev + INDEXING_SUBNETS=subnet-0268b60265d9d6e87,subnet-0c29076fe7de10791,subnet-030a3a47fa10c76b2,subnet-0a6c6c437ed87dda3,subnet-09355979ab5496a50,subnet-0f3a7b40152e63be3 + INDEXING_SECURITY_GROUPS=sg-01817cfe4f3629986 + INDEX_POLL_ENABLED=true + ``` + Leave `SDE_AWS_*` blank so the instance role is used. Keep `SCRAPE_POLL_ENABLED=false` if you only want to + exercise the indexing loop first. +3. Build + migrate + recreate — **order matters; `migrate` is required** (beat rows come from `post_migrate`, + and the production start script does not migrate): + ``` + docker compose -f production.yml build django + docker compose -f production.yml run --rm django python manage.py migrate --noinput + docker compose -f production.yml up -d --force-recreate django celeryworker celerybeat + ``` +4. Verify: + ``` + docker compose -f production.yml run --rm django python manage.py shell -c \ + "from django.conf import settings as s; print(s.INDEXING_DISPATCH_ROLE_ARN, s.INDEXING_SUBNETS, s.INDEX_POLL_ENABLED)" + docker compose -f production.yml run --rm django python manage.py shell -c \ + "from django_celery_beat.models import PeriodicTask; print(list(PeriodicTask.objects.filter(task__startswith='sde_collections.tasks.poll').values_list('name','enabled')))" + docker compose -f production.yml exec celeryworker celery -A config.celery_app inspect ping + ``` + Expect `Poll index runs (every 2 min)` / `Poll crawler S3 results (every 5 min)` with the expected `enabled`. + Confirm migrations `0078`, `0079`, `0080` applied (`showmigrations sde_collections | tail`). + +### C.3 — pre-flight from the staging host +``` +aws s3 cp /etc/hostname s3://sde-cosmos-indexing-dev/curated_collections/_preflight/x/probe.txt +aws s3 cp s3://sde-cosmos-indexing-dev/curated_collections/_preflight/x/probe.txt - +aws s3api list-objects-v2 --bucket sde-cosmos-indexing-dev --prefix index_runs/ # AccessDenied is acceptable +aws sts assume-role --role-arn arn:aws:iam::998871305517:role/CosmosIndexingDispatchRole-dev --role-session-name preflight +``` +Then from a Django shell, on a **scratch** collection (e.g. `config_folder='verify_web'`): +`from sde_collections.indexing.dispatch import run_index_task; run_index_task(c, 'test', 'preflight-1')` +→ expect a `taskArn`; `aws ecs list-tasks --cluster api-scrapers-cluster-dev --family web_cosmos-scraper-dev`. +The task will fail on a missing export — that is the wiring proof. + +### C.4 — closed loop (the indexer's E2E.10) +Pick a small collection **not** in the id-collision set (avoid `gcn_circulars`, `CODE_NASA_API`); the two already +in `sde-web-subset` are `astromaterials_data_system` and `aurorasaurus_reporting_auroras_from_the_ground_up`. +1. Admin → set `workflow_status = Curated`. Confirm: `IndexDispatch` row; objects under + `s3://sde-cosmos-indexing-dev/curated_collections/{cf}/{run_id}/` with `manifest.json` last; ECS task running; + status `TEST_INDEXING`. Watch `/ecs/api-scrapers-dev` in CloudWatch. +2. Wait for `index_runs/{cf}/{run_id}/status.json` (≤ a few minutes). Poller posts `validation.json` to + `#sde-data-curation`; collection **stays** `TEST_INDEXING`. +3. Set `QC Perfect` → prod dispatch (`--target prod`, same collection in dev) → `PRODUCTION_INDEXING` → `PROD_PERFECT`. +4. Failure path: re-curate, delete `manifest.json` before the task reads it (or dispatch against an empty + export) → `INDEXING_FAILED_ON_TEST`, Slack alert. +5. Optional scrape leg: `SCRAPE_POLL_ENABLED=true` + migrate, set a collection to *Ready for Engineering*, + confirm the job JSON lands in the crawler inbox via SSM and the ingest reaches *Ready for Curation*. +6. Tick E2E.2–E2E.10 in the scrapers tracker and "Closed loop verified" / NS.3 in `IMPLEMENTATION_PLAN.md`. + +### Rollback +- Code: `git checkout ` and repeat step 3. Migrations 0078–0080 are additive — leave them applied. +- Disable the pipeline without rolling back: set `INDEX_POLL_ENABLED=false` / `SCRAPE_POLL_ENABLED=false`, + run `migrate` (not just restart — see J), recreate `celerybeat`. Or blank `INDEXING_DISPATCH_ROLE_ARN` to make + dispatch raise on the settings guard. +- Indexer: `git revert -m 1 ` on `develop` and push; CD redeploys. + +--- + +## 8. Cutover and later +- **OOB.1** — add `version: keyword` to live `sde-web` mapping *before* the first write (dynamic mapping would + create it as `text` and the differ never works). Backfilling `version` arms the 431k blast radius — do it with + the deletion guards understood. +- Flip `WEB_INDEX_NAME` `sde-web-subset → sde-web` in `infrastructure/config/settings.py` + task def. No COSMOS change. +- R1 must be fixed before any test/prod tier deploy of the indexer. +- NS.5 — test/prod COSMOS account IDs (`COSMOS_AWS_ACCOUNT_ID[TEST|PROD]`) so the dispatch role + bucket policy + synth for those tiers. +- COSMOS Phase 9 CI/CD (0/8): `validate_deploy_env`/`preflight_aws` commands, `scripts/deploy.sh`, staging/prod + workflows gated on `CD_ENABLED`, `/healthz`, rollback rehearsal. Explicitly not a gate for the loop. +- COSMOS never calls `ecs:DescribeTasks`; a task that dies before writing `status.json` is invisible for + `INDEX_STALL_TIMEOUT_HOURS` (6 h). Consider a `DescribeTasks` check in `poll_index_runs` later. + +--- + +## 9. Doc discrepancies to fix +1. `INDEXING_HANDOFF_TODO.md` / `IMPLEMENTATION_PLAN.md` cite HEAD `9e18ced8` and "uncommitted" state; actual HEAD is `90dbad86`, tree clean. +2. `INDEXING_HANDOFF_TODO.md` says P7 "10 of 14" in one place and "11 of 14" in another. +3. `IMPLEMENTATION_PLAN.md` NS.1 prose and the scrapers tracker still call `i-02b3d3e1ac0671952` (production) "the COSMOS host"; the loop runs on staging. +4. Scrapers `COSMOS_INDEX_REAL_RUN.md` §0 lists NS.3/NS.6 open and COSMOS env "all blank" — NS.6 done 2026-08-20; its worked example uses `nasa_applied_sciences`, which is not in `sde-web-subset`. +5. Deletion guard threshold: `> 0.90` (handoff, tracker, E2E.7a) vs `≥ 0.90` (`COSMOS_INDEX_REAL_RUN.md` §4). Code is `>`. +6. `sde-web-copy` vs `sde-web-subset` — pre-2026-08-19 verifications were against `-copy`; `IMPLEMENTATION_PLAN.md` NS.8 still says so. +7. `sde_collections/DEPLOYMENT.md` says `validate_deploy_env` must refuse identical endpoints; `IMPLEMENTATION_PLAN.md` Phase 9 says tier separation is indexer-side only. +8. `DEPLOYMENT.md` puts the COSMOS ECR repo in "the SMCE account"; all indexing resources are in `998871305517`. +9. Scrapers `infrastructure/DEPLOYMENT.md` `run-task` example lacks the command override. +10. `WORKFLOW.md` omits statuses 22/26 and says the collection moves to QC after validation; the code holds at `TEST_INDEXING` and QC is curator-set. +11. Scrapers `DESIGN.md` (deleted) and `API_SCRAPERS_ARCHITECTURE_REVIEW.md` describe cross-account / 3-source / `us-west-1` — stale. + +--- + +## 10. Open questions for the review +- Should the 12 id-collision collections be enforced in COSMOS code (denylist / override flag) rather than by agreement? +- `export_incomplete` → `succeeded` with `error` (R2): does COSMOS want to treat that as failure, or surface it as a distinct status? +- `tdamm_tag` is exported by COSMOS but dropped by the indexer — intentional for now, or should the mapping carry it? +- Static `SDE_AWS_*` keys vs instance role as the long-term model; env-file source of truth (hand-maintained per host vs SSM Parameter Store / Secrets Manager). +- Do the planning docs with instance IDs / IPs stay in the public repo? +- Fargate networking: dev runs `INDEXING_ASSIGN_PUBLIC_IP=True` on public subnets; is a private subnet + NAT (`False`) the target for prod? +- G, cross-collection duplicates: `BaseUrl.url` is globally unique, so a URL already in another collection's `DumpUrl` rows aborts an ingest. Skip the row (and log), or fail the collection so a curator notices the overlap? +- L: should `ScrapeDispatch` snapshot the previous summary's `LastModified` at dispatch time (migration) so completion detection stops depending on the Django host clock? diff --git a/SQLDumpRestoration.md b/SQLDumpRestoration.md index 866093db..cb545049 100644 --- a/SQLDumpRestoration.md +++ b/SQLDumpRestoration.md @@ -8,6 +8,10 @@ docker-compose -f local.yml run --rm django python manage.py loaddata backup.jso However, if the JSON file is particularly large (>1.5GB), Docker might struggle with this method. In such cases, you can use SQL dump and restore commands as an alternative. +> **Never paste real credentials, hostnames, or database endpoints into this file.** Every value +> below is a placeholder. Read the actual values from the appropriate `.envs/*/.postgres` file on +> the host at the time you run the commands. + ### Steps for Using SQL Dump and Restore 1. Begin by starting only the PostgreSQL container. This prevents the Django container from making changes while the PostgreSQL container is starting up. @@ -21,9 +25,9 @@ docker-compose -f local.yml up postgres ``` $ docker ps CONTAINER ID IMAGE COMMAND -23d33f22cc43 sde_indexing_helper_production_postgres "docker-entrypoint.s…" + "docker-entrypoint.s…" -$ docker exec -it 23d33f22cc43 bash +$ docker exec -it bash ``` 3. Create a connection to the database. @@ -34,25 +38,25 @@ psql -U -d **Note**: - For local deployment, refer to the `.envs/.local/.postgres` file for the `POSTGRES_USER` and `POSTGRES_DB` variables. -- For production deployment, refer to the `.envs/.production/.postgres` file. +- For deployed hosts, refer to that host's `.envs/.production/.postgres` file. 4. Ensure that the database `` is empty. Here's an example: ``` -sde_indexing_helper-# \c -You are now connected to database "sde_indexing_helper" as user "VnUvMKBSdk...". -sde_indexing_helper-# \dt +-# \c +You are now connected to database "" as user "". +-# \dt Did not find any relations. ``` If the database is not empty, delete its contents to create a fresh database: ``` -sde_indexing_helper=# \c postgres //connect to a different database before dropping -You are now connected to database "postgres" as user "VnUvMKBSdk....". -postgres=# DROP DATABASE sde_indexing_helper; +=# \c postgres //connect to a different database before dropping +You are now connected to database "postgres" as user "". +postgres=# DROP DATABASE ; DROP DATABASE -postgres=# CREATE DATABASE sde_indexing_helper; +postgres=# CREATE DATABASE ; CREATE DATABASE ``` @@ -60,7 +64,7 @@ CREATE DATABASE 5. Transfer the backup SQL dump (`backup.sql`) from your local machine to the PostgreSQL container. ``` -docker cp /local/path/backup.sql 23d33f22cc43:/ +docker cp /local/path/backup.sql :/ ``` 6. Import the SQL dump into the PostgreSQL container. @@ -84,109 +88,82 @@ docker-compose -f local.yml run --rm django python manage.py createsuperuser 8. Log in to the COSMOS frontend to ensure that all data has been correctly populated in the UI. +--- +## Making a backup from a deployed host -# making the backup +Read the database values from the host's env file — do not copy them anywhere: ```bash -ssh sde -cat .envs/.production/.postgres +ssh +cat .envs/.production/.postgres # POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD ``` -find the values for the variables: -POSTGRES_HOST=sde-indexing-helper-db.c3cr2yyh5zt0.us-east-1.rds.amazonaws.com -POSTGRES_PORT=5432 -POSTGRES_DB=postgres -POSTGRES_USER=postgres -POSTGRES_PASSWORD=this_is_A_web_application_built_in_2023 +Find the running Postgres container: ```bash docker ps ``` -b3fefa2c19fb +Dump the database. Prefer letting `pg_dump` prompt for the password (`-W`) over putting it in the +command, which would otherwise land in your shell history: -note here that you need to put the ```bash -docker exec -t your_postgres_container_id pg_dump -U your_postgres_user -d your_database_name > backup.sql +docker exec -it pg_dump -h -U -d -W > backup.sql ``` + +### Move the backup to your local machine + ```bash -docker exec -t container_id pg_dump -h host -U user -d database -W > prod_backup.sql +scp :/home/ec2-user/sde-indexing-helper/backup.sql . ``` -docker exec -t b3fefa2c19fb env PGPASSWORD="this_is_A_web_application_built_in_2023" pg_dump -h sde-indexing-helper-db.c3cr2yyh5zt0.us-east-1.rds.amazonaws.com -U postgres -d postgres > prod_backup.sql - -# move the backup to local - go back to local computer and scp the file +To copy it to another host, `scp` or — if the transfer is unreliable — `rsync`: ```bash -scp sde:/home/ec2-user/sde_indexing_helper/prod_backup.sql . +rsync -avzP backup.sql :/home/ec2-user/sde-indexing-helper/ ``` -scp prod_backup.sql sde_staging:/home/ec2-user/sde-indexing-helper -if you have trouble transferring the file, you can use rsync: -rsync -avzP prod_backup.sql sde_staging:/home/ec2-user/sde-indexing-helper/ -# restoring the backup -bring down the local containers +### Restoring the backup + +Bring the local containers down, then start only Postgres: + ```bash docker-compose -f local.yml down docker-compose -f local.yml up postgres docker ps ``` -find the container id - -c11d7bae2e56 - -find the local variables from -cat .envs/.production/.postgres -POSTGRES_HOST=sde-indexing-helper-staging-db.c3cr2yyh5zt0.us-east-1.rds.amazonaws.com -POSTGRES_PORT=5432 -POSTGRES_DB=sde_staging -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres - +Read the target database values from the appropriate env file, then connect and recreate the +database: ```bash -docker exec -it bash -``` -docker exec -it c11d7bae2e56 bash - -## do all the database shit you need to - - +docker exec -it bash psql -U -d -psql -U postgres -d sde_staging -or, if you are on one of the servers: -psql -h sde-indexing-helper-staging-db.c3cr2yyh5zt0.us-east-1.rds.amazonaws.com -U postgres -d postgres +``` +```sql \c postgres -DROP DATABASE sde_staging; -CREATE DATABASE sde_staging; - -# do the backup - -```bash -docker cp prod_backup.sql c11d7bae2e56:/ -docker exec -it c11d7bae2e56 bash +DROP DATABASE ; +CREATE DATABASE ; ``` +Copy the dump into the container and load it: + ```bash +docker cp backup.sql :/ +docker exec -it bash psql -U -d -f backup.sql ``` -psql -U VnUvMKBSdkoFIETgLongnxYHrYVJKufn -d sde_indexing_helper -f prod_backup.sql - -psql -h sde-indexing-helper-staging-db.c3cr2yyh5zt0.us-east-1.rds.amazonaws.com -U postgres -d postgres -f prod_backup.sql -pg_restore -h sde-indexing-helper-staging-db.c3cr2yyh5zt0.us-east-1.rds.amazonaws.com -U postgres -d postgres prod_backup.sql - +Finally, bring everything back up and migrate: -docker down - -docker up build - -migrate - -down +```bash +docker-compose -f local.yml down +docker-compose -f local.yml up --build +docker-compose -f local.yml run --rm django python manage.py migrate +``` -up +**Note:** the `database_backup` and `database_restore` management commands documented in the +[README](./README.md) are the recommended path for routine work; the manual procedure above is for +cases those commands can't handle. diff --git a/WORKFLOW.md b/WORKFLOW.md new file mode 100644 index 00000000..f2f93a0d --- /dev/null +++ b/WORKFLOW.md @@ -0,0 +1,230 @@ +## SDE Data Curation Workflow — Dev Notes + +![SDE Data Curation Workflow (COSMOS + crawl4ai)](./WORKFLOW_DIAGRAM.png) + +> The diagram above is the canonical one-page view of this workflow; this document is its authoritative text. Where the two disagree, this document wins — the known differences are listed in [Where the diagram and this document differ](#where-the-diagram-and-this-document-differ) at the end. Deploy and rollback for the pipeline this workflow describes are in [sde_collections/DEPLOYMENT.md](./sde_collections/DEPLOYMENT.md). + +### Phase 1 — Source Discovery and Setup + +1. The curator identifies a new source for ingestion. + +2. Update the COSMOS collection workflow status to **Research in Progress**. + +3. Finalize all required metadata for the new source, including: + + * `seed_url` + * `division` + * `collection_name` + * Any other required collection metadata + + + +4. Update the workflow status to **Ready for Engineering**. + +5. Send the collection `seed_url` to the scraper running on EC2 (the `sde-crawl4ai-scraper-v1` instance) by delivering a scrape job JSON to its inbox via **AWS SSM Run Command** — the instance has no inbound SSH; SSM is the access path. + +6. When the collection requires changes to the default scraper settings, such as the maximum number of pages, manually add the applicable configuration overrides to the Postgres table through the admin console. + +--- + +### Phase 2 — Scraping and Ingestion + +7. COSMOS generates the job JSON from the collection `seed_url` and any scraper configuration overrides stored in Postgres, and SSM writes it into `jobs/incoming/` on EC2, where the inbox watcher picks it up. + +8. Run the scraper using the delivered job configuration. + +9. Update the collection workflow status based on the scraping result: + + * **Scraping Successful** when scraping completes successfully + * **Scraping Failed** when the scraper encounters an error + + + +10. The scraper saves the scraped data to S3 (`SDE_S3_BUCKET`): documents at `scraped_collections/.json`, failure logs at `failure_logs/_failures.jsonl` and `_failures_summary.json`. + +11. Load the scraped data from S3 into the COSMOS `DumpURL` model. + +--- + +### Phase 3 — Delta Processing and Curation + +12. COSMOS calculates the differences between the latest scraped data and the existing curated data. + +13. Load the calculated differences into the COSMOS `DeltaURL` model. + +14. Update the collection workflow status to **Ready for Curation**. + +15. The curator reviews and curates the delta records. + +16. While curation is underway, update the workflow status to **Curation in Progress**. + +17. After curation is complete, update the workflow status to **Curated**. + +18. The **Curated** status triggers the merge of approved `DeltaURL` records into the `CuratedURL` model. + +--- + +### Phase 4 — Test Indexing and Validation + +19. For collections with the **Curated** status, trigger the indexing pipeline against the test OpenSearch (Serverless) instance — the same chunk → vectorize (SageMaker) → bulk-index pipeline the API scrapers (`sde-api-scrapers`) use. + +20. Index the collection with `public_visibility` set to `true` in the web-document schema. + +21. If test indexing fails, update the workflow status to **Indexing Failed on Test**. + +22. After successful test indexing, trigger the validation script. + +23. The validation script compares the following between the test OpenSearch index and the curated content in COSMOS: + +* Total document count +* Document titles + +24. Post the validation results to the `sde-data-curation` Slack channel. + +25. Based on the validation results, update the collection workflow status to one of the following: + +* **QC: Perfect** +* **QC: Minor Issues** +* **QC: Failed** + +--- + +### Phase 5 — Production Indexing + +26. Create or reference the list of collections that have passed validation with either: +* **QC: Perfect** +* **QC: Minor Issues** + +27. Trigger the indexing pipeline for the validated collections against the production OpenSearch (Serverless) instance. + +28. Keep `public_visibility` set to `true` during production indexing. + +29. If production indexing fails, update the workflow status to **Indexing Failed on Prod**. + +30. After successful production indexing, update the COSMOS collection workflow status to the production outcome that mirrors the QC verdict the collection entered with: + +* **Prod: Perfect** — for collections that entered from **QC: Perfect** +* **Prod: Minor Issues** — for collections that entered from **QC: Minor Issues** + + A collection that passed validation with known minor issues still carries those issues in production, so the production status records that rather than flattening it to "Perfect." Both statuses already exist in COSMOS, and the Slack notification map already covers both transitions. + +--- + +### Phase 6 — Failure Handling and Reprocessing + +31. A developer reviews collections with any of the following failure statuses: +* **Scraping Failed** +* **Indexing Failed on Test** +* **Indexing Failed on Prod** +* **QC: Failed** + +32. The developer identifies and applies the required scraper, configuration, data, or indexing changes. + +33. For scraping failures, rerun the scraper and continue from the scraping and ingestion phase. + +34. For test indexing or validation failures, invoke the indexing pipeline against the test OpenSearch instance with the required updates. + +35. Invoke the validation script again for the QA-failed collections. + +36. Post the updated validation results to the `sde-data-curation` Slack channel. + +37. Repeat the test indexing and validation process until the collection reaches either: + +* **QC: Perfect** +* **QC: Minor Issues** +38. After the collection passes validation, continue with the production indexing process. + +--- + +## Repos for context +- /Users/bbenson/projects/sde-crawl4ai-scraper-v1 +- /Users/bbenson/projects/sde-api-scrapers + +## Workflow Status Progression + +Research in Progress + + ↓ + +Ready for Engineering + + ↓ + +Scraping Successful + + ↓ + +Ready for Curation + + ↓ + +Curation in Progress + + ↓ + +Curated + + ↓ + +Test Indexing + + ↓ + +QC: Perfect / QC: Minor Issues + + ↓ + +Production Indexing + + ↓ + +Prod: Perfect / Prod: Minor Issues + +### Failure Statuses + +Scraping Failed + +Indexing Failed on Test + +Indexing Failed on Prod + +QC: Failed + +Collections in a failure status return to the applicable development, indexing, or validation step after corrective action is completed. + +--- + +## Workflow Status List + +The fourteen statuses a collection is expected to move through, in workflow order, with the `WorkflowStatusChoices` member and stored integer for each (`sde_collections/models/collection_choice_fields.py`). Statuses 21 and 23–25 are new for this pipeline; everything else already exists. + +| Status label | Enum member | Value | Set by | +|---|---|---|---| +| Research in Progress | `RESEARCH_IN_PROGRESS` | 1 | Curator | +| Ready for Engineering | `READY_FOR_ENGINEERING` | 2 | Curator (triggers scrape dispatch) | +| Scraping Successful | `SCRAPING_SUCCESSFUL` | 21 | Ingestion task | +| Scraping Failed | `SCRAPING_FAILED` | 23 | Poller / ingestion task | +| Ready for Curation | `READY_FOR_CURATION` | 4 | Delta migration task | +| Curation in Progress | `CURATION_IN_PROGRESS` | 5 | Curator | +| Curated | `CURATED` | 6 | Curator (triggers promote + test indexing) | +| QC: Failed | `QUALITY_CHECK_FAILED` | 12 | Curator, from the validation report | +| QC: Minor Issues | `QUALITY_CHECK_MINOR` | 18 | Curator, from the validation report | +| QC: Perfect | `QUALITY_CHECK_PERFECT` | 13 | Curator, from the validation report | +| Prod: Minor Issues | `PROD_MINOR` | 15 | Prod indexing task (entered from QC: Minor Issues) | +| Prod: Perfect | `PROD_PERFECT` | 14 | Prod indexing task (entered from QC: Perfect) | +| Indexing Failed on Test | `INDEXING_FAILED_ON_TEST` | 24 | Test indexing task | +| Indexing Failed on Prod | `INDEXING_FAILED_ON_PROD` | 25 | Prod indexing task | + +Two further statuses — **Test Indexing** (22) and **Production Indexing** (26) — exist only while an indexing task is actually running, so a stalled task leaves the collection visibly stuck rather than silently unchanged. They are implementation-internal and deliberately absent from the list above and from the diagram. + +The diagram labels the QC statuses `QC: TEST_FAILED` and `QC: TEST_MINOR_ISSUES`, emphasizing that the QC gate is a judgment on the *test* index. Those are the existing **QC: Failed** (12) and **QC: Minor Issues** (18); no rename is proposed, since both values are live in production data, the Slack map, management commands, and the UI dropdowns. + +--- + +## Where the diagram and this document differ + +Three places where the diagram is loose or wrong. This document is correct in all three; they are recorded so the diagram's wording is not propagated into code or docs later. + +1. **The crawler builds no "config template."** The diagram's Phase 2 step 1 and implementation-flow step 5 say `watch_inbox.sh` creates a config from the seed URL and overrides. It does not — `watch_inbox.sh` only watches the inbox with inotify and launches `run.py` under `flock`. `run.py` merges the job JSON's non-null overrides onto the crawler's own defaults; there is no separate template file or templating step. +2. **Phase 6 is broader than the diagram's box.** The diagram's Phase 6 shows only the indexing failure statuses and the loop back through validation. The failure-handling phase above also covers **Scraping Failed** and **QC: Failed**, which re-enter the pipeline at the scraping and test-indexing phases respectively. +3. **Phase 3's step numbering slips in the diagram.** Its steps 3 and 4 both land on "Curation in Progress," and the transition to **Curated** is only implied by step 5's merge. The authoritative sequence is steps 12–18 above: deltas loaded → Ready for Curation → Curation in Progress → Curated → merge into `CuratedUrl`. diff --git a/WORKFLOW_DIAGRAM.png b/WORKFLOW_DIAGRAM.png new file mode 100644 index 00000000..10fe1548 Binary files /dev/null and b/WORKFLOW_DIAGRAM.png differ diff --git a/bandit-config.yml b/bandit-config.yml index 36a65525..a6e47d5d 100644 --- a/bandit-config.yml +++ b/bandit-config.yml @@ -1,26 +1,12 @@ # bandit-config.yml -skips: - - B101 # Skip assert used (often used in tests) - - B403 # Skip import from the pickle module - -exclude: - - ./tests/ # Exclude test directories - - ./migrations/ # Exclude migration directories - - ./venv/ # Exclude virtual environment - +# Only run these checks (same effective profile as the previous legacy config): +# B105 - hardcoded password strings +# B602 - subprocess call with shell=True tests: - - B105 # Include test for hardcoded password strings - - B602 # Include test for subprocess call with shell equals true - -profiles: - default: - include: - - B403 # Include test for dangerous default argument - exclude: - - B401 # Exclude test for import telnetlib - -# Set the severity level to focus on higher-risk issues -severity: 'HIGH' + - B105 + - B602 -# Set the confidence level to ensure that reported issues are likely true positives -confidence: 'HIGH' +exclude_dirs: + - tests + - migrations + - venv diff --git a/config/settings/base.py b/config/settings/base.py index 14e0ad28..843e4bcb 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -338,19 +338,50 @@ "EXCEPTION_HANDLER": "sde_indexing_helper.utils.exceptions.custom_exception_handler", } -GITHUB_ACCESS_TOKEN = env("GITHUB_ACCESS_TOKEN") -SINEQUA_CONFIGS_GITHUB_REPO = env("SINEQUA_CONFIGS_GITHUB_REPO") -SINEQUA_CONFIGS_REPO_MASTER_BRANCH = env("SINEQUA_CONFIGS_REPO_MASTER_BRANCH") -SINEQUA_CONFIGS_REPO_DEV_BRANCH = env("SINEQUA_CONFIGS_REPO_DEV_BRANCH") -SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH = env("SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH") SLACK_WEBHOOK_URL = env("SLACK_WEBHOOK_URL") -XLI_USER = env("XLI_USER") -XLI_PASSWORD = env("XLI_PASSWORD") -LRM_DEV_USER = env("LRM_DEV_USER") -LRM_DEV_PASSWORD = env("LRM_DEV_PASSWORD") -LRM_QA_USER = env("LRM_QA_USER") -LRM_QA_PASSWORD = env("LRM_QA_PASSWORD") -LRM_DEV_TOKEN = env("LRM_DEV_TOKEN") -XLI_TOKEN = env("XLI_TOKEN") INFERENCE_API_URL = env("INFERENCE_API_URL", default="http://host.docker.internal:8000") TDAMM_CLASSIFICATION_THRESHOLD = env("TDAMM_CLASSIFICATION_THRESHOLD", default="0.5") + +# --- SDE curation pipeline --- +AWS_REGION = env("AWS_REGION", default="us-east-1") +SDE_S3_BUCKET = env("SDE_S3_BUCKET", default="") # crawler output bucket +CRAWLER_INSTANCE_ID = env("CRAWLER_INSTANCE_ID", default="") # i-0b6a61d95888886f4 on dev +CRAWLER_INBOX_PATH = env("CRAWLER_INBOX_PATH", default="/opt/sde-crawler/jobs/incoming") +SCRAPE_POLL_ENABLED = env.bool("SCRAPE_POLL_ENABLED", default=False) +# A dispatched crawl with no fresh S3 summary after this long is declared dead (P4). +SCRAPE_STALL_TIMEOUT_HOURS = env.int("SCRAPE_STALL_TIMEOUT_HOURS", default=24) +INFERENCE_ENABLED = env.bool("INFERENCE_ENABLED", default=False) +# pipeline-scoped credentials for local dev ONLY; blank in AWS (instance role takes over) +SDE_AWS_ACCESS_KEY_ID = env("SDE_AWS_ACCESS_KEY_ID", default="") +SDE_AWS_SECRET_ACCESS_KEY = env("SDE_AWS_SECRET_ACCESS_KEY", default="") +# only needed with temporary creds (e.g. `aws configure export-credentials --profile sde-dev`) +SDE_AWS_SESSION_TOKEN = env("SDE_AWS_SESSION_TOKEN", default="") +# COSMOS never talks to OpenSearch or SageMaker: chunk/vectorize/index AND the QC validation +# report are produced by the WEB_COSMOS task in sde-api-scrapers (branch web-indexing), which +# holds the AOSS credentials. COSMOS only writes exports to S3, assumes one role, and calls +# ecs:RunTask. +# +# --- P7 indexing hand-off --- +# ONLY the sde-dev environment is wired today: the indexer's dev deployment targets the +# disposable sde-web-copy index, and its target->endpoint resolution is tier-capped, so a +# dev dispatch can never reach prod AOSS. All defaults are blank/off — with any of the +# required values unset, dispatch fails fast and the poller never runs. Dev values: +# SDE_INDEX_BUCKET sde-cosmos-indexing-dev +# INDEXING_ECS_CLUSTER api-scrapers-cluster-dev +# INDEXING_TASK_FAMILY web_cosmos-scraper-dev +# INDEXING_CONTAINER_NAME WEB_COSMOSContainer +# INDEXING_DISPATCH_ROLE_ARN arn:aws:iam:::role/CosmosIndexingDispatchRole-dev +SDE_INDEX_BUCKET = env("SDE_INDEX_BUCKET", default="") # distinct from SDE_S3_BUCKET (crawler) +INDEXING_ECS_CLUSTER = env("INDEXING_ECS_CLUSTER", default="") +INDEXING_TASK_FAMILY = env("INDEXING_TASK_FAMILY", default="") +INDEXING_CONTAINER_NAME = env("INDEXING_CONTAINER_NAME", default="WEB_COSMOSContainer") +# Blank = RunTask with the pipeline session's own creds (local dev only; the role's trust +# policy admits just the instance role, so a laptop SSO session can never assume it). +INDEXING_DISPATCH_ROLE_ARN = env("INDEXING_DISPATCH_ROLE_ARN", default="") +# Fargate RunTask needs awsvpc network config; comma-separated ids, from the indexer's VPC. +INDEXING_SUBNETS = env("INDEXING_SUBNETS", default="") +INDEXING_SECURITY_GROUPS = env("INDEXING_SECURITY_GROUPS", default="") +INDEXING_ASSIGN_PUBLIC_IP = env.bool("INDEXING_ASSIGN_PUBLIC_IP", default=True) +INDEX_POLL_ENABLED = env.bool("INDEX_POLL_ENABLED", default=False) +# An index run with no status.json after this long is declared dead (indexing is minutes, not hours). +INDEX_STALL_TIMEOUT_HOURS = env.int("INDEX_STALL_TIMEOUT_HOURS", default=6) diff --git a/config/settings/test.py b/config/settings/test.py index d7eaa130..72d18875 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -28,5 +28,21 @@ # DEBUGGING FOR TEMPLATES # ------------------------------------------------------------------------------ TEMPLATES[0]["OPTIONS"]["debug"] = True # type: ignore # noqa F405 + +# CELERY +# ------------------------------------------------------------------------------ +# Never publish to the real broker from tests. The container entrypoint exports +# CELERY_BROKER_URL=${REDIS_URL} — the same Redis the celeryworker container consumes — +# so any test that changes a workflow status without patching .delay() would enqueue a +# real message, and the live worker would run it against the LOCAL database (test-DB +# collection ids can collide with local rows, flipping their statuses). kombu's +# memory:// transport queues in-process and dies with the test run. +# Celery gives the CELERY_BROKER_URL *environment variable* precedence over Django +# settings, so the env var must be overridden too; this runs at settings import, before +# the celery app's lazy config finalizes. +import os # noqa: E402 + +os.environ["CELERY_BROKER_URL"] = "memory://" +CELERY_BROKER_URL = "memory://" # Your stuff... # ------------------------------------------------------------------------------ diff --git a/config_generation/README.md b/config_generation/README.md deleted file mode 100644 index 10edb8ff..00000000 --- a/config_generation/README.md +++ /dev/null @@ -1,18 +0,0 @@ -## Access Tokens -Generate your access token using https://doc.sinequa.com/en.sinequa-es.v11/Content/en.sinequa-es.how-to.access-tokens.html, then add it to a config.py as `token = ` - -## General API Stuff -https://doc.sinequa.com/en.sinequa-es.v11/Content/en.sinequa-es.devDoc.webservice.rest.html - -## Query API -For extra details on how to use the query api, you can see - -## Indexing API -Don't be fooled by the page on indexing.... -https://doc.sinequa.com/en.sinequa-es.v11/Content/en.sinequa-es.devDoc.webservice.rest-indexing.html#indexing-collection - -You want the page on jobs https://doc.sinequa.com/en.sinequa-es.v11/Content/en.sinequa-es.devDoc.webservice.rest-operation.html#operationcollectionStart. - -## Creating Job Lists -Update config.py to contain the latest collections you want to index. Then run generate_jobs.py and it will create the parallel batches. -If you want it to run on multiple nodes, you will need to add that in two places in the file, and then you won't be able to run the lists from the masterlist, because of a sinequa bug. diff --git a/config_generation/api.py b/config_generation/api.py deleted file mode 100644 index 7ae4d99b..00000000 --- a/config_generation/api.py +++ /dev/null @@ -1,88 +0,0 @@ -from typing import Any - -import requests - -from config import tokens - -server_configs: dict[str, dict[str, str]] = { - "ren_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://sde-renaissance.nasa-impact.net", - }, - "test_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://10.51.14.135", - }, -} - - -class Api: - def __init__(self, server_name: str) -> None: - self.headers: dict[str, str] = {"Authorization": f"Bearer {tokens[server_name]}"} - self.app_name: str = server_configs[server_name]["app_name"] - self.query_name: str = server_configs[server_name]["query_name"] - self.base_url: str = server_configs[server_name]["base_url"] - - def process_response(self, url: str, payload: dict[str, Any]) -> dict[str, Any]: - response = requests.post(url, headers=self.headers, json=payload, verify=False) - - if response.status_code == 200: - meaningful_response = response.json() - else: - meaningful_response = {"response_text": response.text} - - return meaningful_response - - def query(self, term: str): - url = f"{self.base_url}/api/v1/search.query" - payload = { - "app": self.app_name, - "query": { - "name": self.query_name, - "action": "search", - "text": term, - "pageSize": 1000, - "tab": "all", - }, - "pretty": "true", - } - - return self.process_response(url, payload) - - def sql(self, source: str, collection: str = "", fetch_all: bool = False) -> dict[str, Any]: - url = f"{self.base_url}/api/v1/engine.sql" - - collection_name = f"/{source}/{collection}/" - sql_command_all = "select url1,title,collection from @@ScienceMissionDirectorate" - if fetch_all: - sql_command = sql_command_all - else: - sql_command = f"{sql_command_all} where collection='{collection_name}'" - - payload = { - "sql": sql_command, - "maxRows": 10000000, # ten million - "pretty": "true", - } - response = self.process_response(url, payload) - - return response - - def run_indexer(self, source_name: str, collection_name: str) -> dict[str, Any]: - """Starts indexing on the given collection. Equivalent to pressing the play button in the - interface. This function will return the response from the sinequa server and then the - server will run the collection on it's own without restraining the python execution. - - Args: - source_name (str): this is the name of the source in sinequa, for example, Scraping or SMD - collection_name (str): for example astro_home_page - """ - url = f"{self.base_url}/api/v1/operation.collectionStart" - - payload = { - "collection": f"/{source_name}/{collection_name}/", - } - - return self.process_response(url, payload) diff --git a/config_generation/config_example.py b/config_generation/config_example.py deleted file mode 100644 index c1d9310b..00000000 --- a/config_generation/config_example.py +++ /dev/null @@ -1,69 +0,0 @@ -from sources_to_scrape import sources_to_index_test_grid_20240809 - -tokens: dict[str, str] = { - "test_server": "token here", - "ren_server": "token here", -} - -AVAILABLE_INDEXERS_TEST = [ - "IndexerServerA/identity0", - "IndexerServerB/identity0", -] - -AVAILABLE_INDEXERS_PROD = ["NodeINDEX1/identity0", "NodeINDEX2/identity0"] - -TEST_SERVER_INDEXES = [ # this is the test server list - # "sde_neural_test_index", - "sde_index" -] - -PROD_SERVER_INDEXES = [ - # "EDP_Audit_1", - # "SMD_LSDA_Repository_1", - # # "EDP_UserMetadata_1", - # "SMD_NTRS_Repository_1", - # "GCMD_Repository_1", - # "SMD_PLANETARY_Repository_1", - # # "GCMD_Repository_1_Metadata", - # "SMD_PLANETARY_Repository_2", - # "GCMD_Repository_2", - # "STI_Repository_1", - # # "GCMD_Repository_3_Metadata", - # # "STI_Repository_1_Metadata", - # "HELIO_Repository_1", - # "STI_Repository_2", - "SDE_Index", - # "STI_Repository_2_Metadata", - # "SMD_ASTRO_Repository_1", - # "STI_Repository_3", - # "SMD_ASTRO_Repository_2", - # "STI_Repository_4", - # "SMD_EARTHSCIENCE_Repository_1", - # # "SinequaDoc", - # "SMD_GENELAB_Repository_1", - # "Test", -] - - -SERVER_INFO = { - "test": { - "indexes": TEST_SERVER_INDEXES, - "indexers": AVAILABLE_INDEXERS_TEST, - }, - "prod": { - "indexes": PROD_SERVER_INDEXES, - "indexers": AVAILABLE_INDEXERS_PROD, - }, -} - -# Job Creation Config -collection_list: list[str] = sources_to_index_test_grid_20240809 # python list -date = "20240809" -source = "SDE" -server = "test" - -# auto assigned -batch_delete_name: str = f"sources_to_delete_on_{server}_{date}" -batch_index_name: str = f"sources_to_index_on_{server}_{date}" -available_indexers = SERVER_INFO[server]["indexers"] -indexes_to_delete_from = SERVER_INFO[server]["indexes"] diff --git a/config_generation/db_to_xml.py b/config_generation/db_to_xml.py deleted file mode 100644 index 89de197f..00000000 --- a/config_generation/db_to_xml.py +++ /dev/null @@ -1,442 +0,0 @@ -import json -import xml.etree.ElementTree as ET - -import xmltodict - -from sde_collections.models.collection_choice_fields import ( - ConnectorChoices, - DocumentTypes, -) - - -class XmlEditor: - """ - Class is instantiated with a path to an xml. - An internal etree is generated, and changes are made in place. - An ouput path is given and the etree is saved to it. - """ - - def __init__(self, xml_string: str): - self.xml_tree = self._get_tree(xml_string) - - def _get_tree(self, xml_string) -> ET.ElementTree: - """takes the path of an xml file and opens it as an ElementTree object""" - return ET.ElementTree(ET.fromstring(xml_string)) - - def get_tag_value(self, tag_name: str, strict: bool = False) -> str | list[str]: - """ - Retrieves the value of the specified XML tag. If 'strict' is True, the function will - raise an error if more than one value is found, and it will return the single value. - - Parameters: - - tag_name (str): Can be either the top level tag or a path specifying a child tag, e.g., 'parent/child'. - - strict (bool): If True, raises an error when more than one value is found, or if no values are found. - - Returns: - - str: The text of the single XML element matching the tag_name if strict is True and exactly one match exists. - - Raises: - - ValueError: If 'strict' is True and either no values or more than one value is found. - """ - - elements = self.xml_tree.findall(tag_name) - if strict: - if len(elements) == 0: - raise ValueError(f"No elements found for the tag '{tag_name}'") - elif len(elements) > 1: - raise ValueError(f"Multiple elements found for the tag '{tag_name}': expected exactly one.") - return elements[0].text - else: - return [element.text for element in elements] - - def _add_declaration(self, xml_string: str): - """adds xml declaration to xml string""" - declaration = """\n""" - - return declaration + xml_string - - def update_config_xml(self): - xml_string = ET.tostring( - self.xml_tree.getroot(), - encoding="utf8", - method="html", - xml_declaration=True, - ) - xml_string = xml_string.decode("utf-8") - - xml_string = self._add_declaration(xml_string) - xml_string = self._resave_pretty(xml_string) - - return xml_string - - def _resave_pretty(self, xml_string): - """opens and resaves a file to reformat it""" - - xml = xmltodict.parse(xml_string) - return xmltodict.unparse(xml, pretty=True) - - def update_or_add_element_value(self, path, new_value, parent_element_name=None): - """ - Update or create a value in the XML. - - Parameters: - - xml_string (str): The original XML string. - - path (str): The path to the element. Elements are separated by '/'. Example: 'root/child/grandchild' - - new_value (str): The new value to set. - - Returns: - - str: The updated XML string. - """ - - # Parse the XML string into an ElementTree - root = self.xml_tree.getroot() - - # Split the path into its components - if parent_element_name is not None: - path = f"{parent_element_name}/{path}" - elements = path.split("/") - - # Traverse and/or create the path - current_element = root - for element in elements: # Skip the root element - # If the child exists, move to it; otherwise, create it - next_element = current_element.find(element) - if next_element is None: - next_element = ET.SubElement(current_element, element) - current_element = next_element - - # Set the value - current_element.text = new_value - - # Return the updated XML as a string - return ET.tostring(root, encoding="utf-8").decode("utf-8") - - def convert_indexer_to_scraper(self) -> None: - """ - assuming this class has been instantiated with a previously constructed indexer config - some values must now be modified so it will be an effective scraper - """ - self.update_or_add_element_value("Indexers", "") - self.update_or_add_element_value("Plugin", "SMD_Plugins/Sinequa.Plugin.ListCandidateUrls") - self.update_or_add_element_value("ShardIndexes", "") - self.update_or_add_element_value("ShardingStrategy", "") - self.update_or_add_element_value("WorkerCount", "8") - self.update_or_add_element_value("LogLevel", "0", parent_element_name="System") - self.update_or_add_element_value("Simulate", "true", parent_element_name="IndexerClient") - - def convert_scraper_to_indexer(self) -> None: - # this is specialized for the production instance right now - self.update_or_add_element_value("Indexers", "") - self.update_or_add_element_value("Plugin", "") - self.update_or_add_element_value("Identity", "NodeIndexer1/identity0") # maybe make this blank? - self.update_or_add_element_value("ShardIndexes", "") - self.update_or_add_element_value("ShardingStrategy", "") - self.update_or_add_element_value("WorkerCount", "8") - self.update_or_add_element_value("LogLevel", "20", parent_element_name="System") - self.update_or_add_element_value("Simulate", "false", parent_element_name="IndexerClient") - - def convert_template_to_scraper(self, collection) -> None: - """ - assuming this class has been instantiated with the scraper_template.xml - """ - self.update_or_add_element_value("Url", collection.url) - - self.update_or_add_element_value("TreeRoot", collection.tree_root) - if collection.document_type: - self.add_document_type_mapping(document_type=collection.get_document_type_display(), criteria=None) - - scraper_config = self.update_config_xml() - return scraper_config - - def convert_template_to_job(self, collection, job_source) -> None: - """ - assuming this class has been instantiated with the job_template.xml - """ - self.update_or_add_element_value("Collection", f"/{job_source}/{collection.config_folder}/") - job_config = self.update_config_xml() - return job_config - - def convert_template_to_indexer(self, scraper_editor) -> None: - """ - assuming this class has been instantiated with the final_config_template.xml - """ - - transfer_fields = [ - "Throttle", - ] - - double_transfer_fields = [ - ("UrlAccess", "UseBrowserForWebRequests"), - ("UrlAccess", "BrowserForWebRequestsReadinessThreshold"), - ("UrlAccess", "BrowserForWebRequestsInitialDelay"), - ("UrlAccess", "BrowserForWebRequestsMaxTotalDelay"), - ("UrlAccess", "BrowserForWebRequestsMaxResourcesDelay"), - ("UrlAccess", "BrowserForWebRequestsLogLevel"), - ("UrlAccess", "BrowserForWebRequestsViewportWidth"), - ("UrlAccess", "BrowserForWebRequestsViewportHeight"), - ("UrlAccess", "BrowserForWebRequestsAdditionalJavascript"), - ("UrlAccess", "PostLoginUrl"), - ("UrlAccess", "PostLoginData"), - ("UrlAccess", "GetBeforePostLogin"), - ("UrlAccess", "PostLoginAutoRedirect"), - ("UrlAccess", "ReLoginCount"), - ("UrlAccess", "ReLoginDelay"), - ("UrlAccess", "DetectHtmlLoginPattern"), - ("IndexerClient", "RetryTimeout"), - ("IndexerClient", "RetrySleep"), - ] - - triple_transfer_fields = [ - ("UrlAccess", "BrowserLogin", "Activate"), - ("UrlAccess", "BrowserLogin", "RemoteDebuggingPort"), - ("UrlAccess", "BrowserLogin", "BrowserLogLevel"), - ("UrlAccess", "BrowserLogin", "ShowDevTools"), - ("UrlAccess", "BrowserLogin", "SuccessCondition"), - ("UrlAccess", "BrowserLogin", "CookieFilter"), - ] - - for field in transfer_fields: - self.update_or_add_element_value(field, scraper_editor.get_tag_value(field, strict=True)) - - for parent, child in double_transfer_fields: - self.update_or_add_element_value( - f"{parent}/{child}", scraper_editor.get_tag_value(f"{parent}/{child}", strict=True) - ) - - for grandparent, parent, child in triple_transfer_fields: - self.update_or_add_element_value( - f"{grandparent}/{parent}/{child}", - scraper_editor.get_tag_value(f"{grandparent}/{parent}/{child}", strict=True), - ) - - scraper_config = self.update_config_xml() - return scraper_config - - def _mapping_exists(self, new_mapping: ET.Element): - """ - Check if the mapping with given parameters already exists in the XML tree - """ - xml_root = self.xml_tree.getroot() - - for mapping in xml_root.findall("Mapping"): - existing_mapping = {child.tag: (child.text if child.text is not None else "") for child in mapping} - new_mapping_dict = {child.tag: (child.text if child.text is not None else "") for child in new_mapping} - if existing_mapping == new_mapping_dict: - return True - - return False - - @staticmethod - def _standardize_selection(selection): - """ - some existing selections may use double quotes while new ones need to use single quotes - # prior rule generations were not as selective, so some old selections used a trailing * - # while the new selection will not - this function creates two selections that will match against the old format and allow it to - be replaced by the _generic_mapping function - """ - standardized_quotes = selection.replace('"', "'") - # standardized_quotes_less_selective = standardized_quotes.replace( - # "*'", "'" - # ) - - return list(set(selection, standardized_quotes)) # , standardized_quotes_less_selective) - - def _generic_mapping( - self, - name: str = "", - description: str = "", - value: str = "", - selection: str = "", - ): - """ - most mappings take the same fields, so this gives a generic way to make a mapping - """ - xml_root = self.xml_tree.getroot() - - existing_mapping = None - for mapping in xml_root.findall("Mapping"): - mapping_name = mapping.find("Name") - mapping_selection = mapping.find("Selection") - - if ( - mapping_name - and mapping_name.text == name - and mapping_selection - and mapping_selection.text in self._standardize_selection(selection) - ): - existing_mapping = mapping - break - - if existing_mapping: - # If an existing mapping is found, overwrite its values - existing_mapping_value = existing_mapping.find("Value") - if existing_mapping_value: - existing_mapping_value.text = value - else: - # If no existing mapping is found, create a new one - mapping = ET.Element("Mapping") - ET.SubElement(mapping, "Name").text = name - ET.SubElement(mapping, "Description").text = description - ET.SubElement(mapping, "Value").text = value - ET.SubElement(mapping, "Selection").text = selection - ET.SubElement(mapping, "DefaultValue").text = "" - xml_root.append(mapping) - - def add_document_type_mapping(self, document_type: str, criteria: str) -> None: - if criteria: - selection = f"doc.url1 match '{criteria}'" - else: - selection = "" - self._generic_mapping( - name="sourcestr56", - value=f'"{document_type}"', - selection=selection, - ) - - def add_title_mapping(self, title_value: str, title_criteria: str) -> None: - title_criteria = title_criteria.rstrip("/") - sinequa_code_markers = ["xpath", "Concat", "IfEmpty", "doc.title", "doc.url1"] - if not any(marker in title_value for marker in sinequa_code_markers): - # exact title replacements need quotes - # sinequa code needs to NOT have quotes - title_value = f'"{title_value}"' - - self._generic_mapping( - name="title", - value=title_value, - selection=f"doc.url1 match '{title_criteria}'", - ) - - def add_job_list_item(self, job_name): - """ - this is specifically for editing joblist templates by adding a new collection to a joblist - config_generation/xmls/joblist_template.xml - """ - xml_root = self.xml_tree.getroot() - - mapping = ET.Element("JobListItem") - ET.SubElement(mapping, "Name").text = job_name - ET.SubElement(mapping, "StopOnError").text = "false" - xml_root.append(mapping) - - def add_id(self) -> None: - self._generic_mapping( - name="id", - value="doc.url1", - ) - - def add_document_type(self, document_type: str) -> None: - self._generic_mapping( - name="sourcestr56", - value=f'"{document_type}"', - ) - - def add_xpath_indexing_filter(self, xpath: str, selection: str = "") -> None: - # TODO: take in selection as an arg - """filters out the content of an xpath from being indexed along with the document""" - - xml_root = self.xml_tree.getroot() - - mapping = ET.Element("IndexingFilter") - ET.SubElement(mapping, "XPath").text = xpath - ET.SubElement(mapping, "IncludeMode").text = "false" - ET.SubElement(mapping, "Selection").text = selection - xml_root.append(mapping) - - def add_url_exclude(self, url_pattern: str) -> None: - """ - excludes a url or url pattern, such as - - https://webb.nasa.gov/content/forEducators/realworld* - - https://webb.nasa.gov/content/features/index.html - - *.rtf - """ - - xml_root = self.xml_tree.getroot() - - for url_index_excluded in xml_root.findall("UrlIndexExcluded"): - if url_index_excluded.text == url_pattern: - return # stop the function if the url pattern already exists - - # add the url pattern if it doesn't already exist - ET.SubElement(xml_root, "UrlIndexExcluded").text = url_pattern - - def add_url_include(self, url_pattern: str) -> None: - """ - includes a url or url pattern, such as - - https://webb.nasa.gov/content/forEducators/realworld* - - https://webb.nasa.gov/content/features/index.html - - *.rtf - I'm not sure if exclusion rules override includes or if includes override - exclusion rules. - """ - - xml_root = self.xml_tree.getroot() - - for url_index_included in xml_root.findall("UrlIndexIncluded"): - if url_index_included.text == url_pattern: - return # stop the function if the url pattern already exists - - # add the url pattern if it doesn't already exist - ET.SubElement(xml_root, "UrlIndexIncluded").text = url_pattern - - def _find_treeroot_field(self): - treeroot = self.xml_tree.find("TreeRoot") - if treeroot is None: - treeroot = self.xml_tree.find("treeRoot") - return treeroot - - def fetch_treeroot(self): - treeroot = self._find_treeroot_field() - return treeroot.text - - def fetch_division_name(self): - # this is pretty brittle and can break easily if the treeRoot field is changed - treeroot = self.fetch_treeroot() - splits = [split for split in treeroot.split("/") if split] - try: - division, name = splits - except ValueError: - print(f"Could not find division and name in {treeroot}") - division = "" - name = "" - return division, name - - def fetch_url(self): - url = self.xml_tree.find("Url") - if url is None: - url = self.xml_tree.find("url") - try: - return url.text - except AttributeError: - return "" - - def fetch_document_type(self): - DOCUMENT_TYPE_COLUMN = "sourcestr56" - try: - document_type_text = self.xml_tree.find(f"Mapping[Name='{DOCUMENT_TYPE_COLUMN}']/Value").text - except AttributeError: - return None - - try: - document_type = DocumentTypes.lookup_by_text(json.loads(document_type_text)) - except json.decoder.JSONDecodeError: - document_type = None - return document_type - - def fetch_connector(self): - connector = self.xml_tree.find("Connector") - if connector is None: - connector = self.xml_tree.find("connector") - - if connector is None: - connector = ConnectorChoices.NO_CONNECTOR - elif connector.text.strip() == "crawler2": - connector = ConnectorChoices.CRAWLER2 - elif connector.text.strip() == "json": - connector = ConnectorChoices.JSON - elif connector.text.strip() == "hyperindex": - connector = ConnectorChoices.HYPERINDEX - else: # as a catch all - connector = ConnectorChoices.NO_CONNECTOR - return connector diff --git a/config_generation/db_to_xml_file_based.py b/config_generation/db_to_xml_file_based.py deleted file mode 100644 index 14b077b7..00000000 --- a/config_generation/db_to_xml_file_based.py +++ /dev/null @@ -1,119 +0,0 @@ -# This file uses the previously written file-based xml handling code to supplement -# the new github-based xml code so that the job-creation pipeline will still work -# we can remove it once we incorporate job creation into the github pipeline -import os -import xml.etree.ElementTree as ET - - -class XmlEditor: - """ - Class is instantiated with a path to an xml. - An internal etree is generated, and changes are made in place. - An ouput path is given and the etree is saved to it. - """ - - def __init__(self, xml_path: str): - self.input_path = xml_path - self.xml_tree = self._get_tree(self.input_path) - - def _get_tree(self, xml_path) -> ET.ElementTree: - """takes the path of an xml file and opens it as an ElementTree object""" - return ET.parse(xml_path) - - def _add_declaration(self, output_path: str): - """opens an existing file and adds a declaration""" - declaration = """""" - with open(output_path, "r+") as f: - content = f.read() - f.seek(0, 0) - f.write(declaration.rstrip("\r\n") + "\n" + content) - - def _update_config_xml(self, output_path: str): - self.xml_tree.write( - output_path, - method="html", - encoding="utf-8", - xml_declaration=True, - ) - - self._add_declaration(output_path) - self._resave_pretty(output_path) - - def _resave_pretty(self, output_path): - """opens and resaves a file to reformat it""" - import xmltodict - - with open(output_path) as f: - xml_data = f.read() - xml = xmltodict.parse(xml_data) - with open(output_path, "w") as f: - f.write(xmltodict.unparse(xml, pretty=True)) - - def create_folder_if_needed(self, folder_path: str): - """ - sinequa configs are source_name/collection_name/default.xml - this function helps make the collection_name folder - folder path is a full exact path ending in a potential collection_name folder - """ - - try: - os.makedirs(folder_path) - except FileExistsError: - pass - except OSError as error: - print(f"Error creating folder '{folder_path}': {error}") - - def create_config_folder_and_default(self, source_name, collection_name): - """ - sinequa configs are source_name/collection_name/default.xml - makes a folder named after the collection with a default.xml inside of it - does this inside of the the specified source folder - """ - - # Create a folder named after source inside the desired directory - config_folder_path = os.path.join(source_name, collection_name) - self.create_folder_if_needed(config_folder_path) - xml_path = os.path.join(config_folder_path, "default.xml") - - # self._write_xml(xml_path) - self._update_config_xml(xml_path) - - def update_or_add_element_value( - self, - element_name: str, - element_value: str, - parent_element_name: str = "", - add_duplicate: bool = False, - ) -> None: - """can update the value of either a top level or secondary level value in the sinequa config - - Args: - element_name (str): name of the sinequa element, such as "Simulate" - element_value (str): value to be stored to element, such as "false" - parent_element_name (str, optional): parent of the element, such as "IndexerClient" - Defaults to None. - """ - - xml_root = self.xml_tree.getroot() - parent_element = xml_root if not parent_element_name else xml_root.find(parent_element_name) - - if parent_element is None: - raise ValueError(f"Parent element '{parent_element_name}' not found in XML.") # noqa: E713 - - existing_element = parent_element.find(element_name) - if not add_duplicate and existing_element: - existing_element.text = element_value - else: - ET.SubElement(parent_element, element_name).text = element_value - - def add_job_list_item(self, job_name): - """ - this is specifically for editing joblist templates by adding a new collection to a joblist - config_generation/xmls/joblist_template.xml - """ - xml_root = self.xml_tree.getroot() - - mapping = ET.Element("JobListItem") - ET.SubElement(mapping, "Name").text = job_name - ET.SubElement(mapping, "StopOnError").text = "false" - xml_root.append(mapping) diff --git a/config_generation/delete_config_folders.py b/config_generation/delete_config_folders.py deleted file mode 100644 index 119d48fc..00000000 --- a/config_generation/delete_config_folders.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -this file deletes the files associated with a folder name as defined in config.py -it will delete - - config folders - - commands - - jobs -""" - -import glob -import os -import shutil - -from config import collection_list as collection_names -from config import source - - -def delete_folders_by_name(collection_names, directory): - for root, dirs, files in os.walk(directory, topdown=False): - for name in dirs: - if name in collection_names: - folder_path = os.path.join(root, name) - shutil.rmtree(folder_path) - print(f"Deleted folder: {folder_path}") - - -def delete_xml_files_by_name(collection_names, directory): - """ - this deletes files that match the sinequa pattern of - command.collectioncache.SMD.PDS_Users_Guides_Website.xml - where the name is surrounded by periods. this will prevent accidental matches to - similar names, but may leave a few stray undeleted files - """ - # Define the pattern to match the files - for collection_name in collection_names: - pattern = f"*.{collection_name}.xml" - - # Use glob to find all files in the directory that match the pattern - for file_path in glob.glob(os.path.join(directory, pattern)): - try: - os.remove(file_path) - print(f"Deleted file: {file_path}") - except OSError as e: - print(f"Error deleting file {file_path}: {e.strerror}") - - -delete_folders_by_name(collection_names, f"../sinequa_configs/sources/{source}/") -delete_xml_files_by_name( - collection_names, "../sinequa_configs/commands/" -) # this might delete jobs from any source, however not a huge issue right now -delete_xml_files_by_name( - collection_names, "../sinequa_configs/jobs/" -) # this might delete jobs from any source, however not a huge issue right now diff --git a/config_generation/delete_server_content.py b/config_generation/delete_server_content.py deleted file mode 100644 index a9cf6bcf..00000000 --- a/config_generation/delete_server_content.py +++ /dev/null @@ -1,25 +0,0 @@ -"""this file uses data from config.py to generate a command that will delete collection content from multiple indexes""" - -from db_to_xml_file_based import XmlEditor - -from config import ( - batch_delete_name, - collection_list, - engines, - indexes_to_delete_from, - source, -) - -COMMAND_FILES_PATH = "../sinequa_configs/commands/" -DELETE_COMMAND_TEMPLATE_PATH = "xmls/delete_template.xml" - -command_file = XmlEditor(DELETE_COMMAND_TEMPLATE_PATH) - -command_file.update_or_add_element_value(element_name="Engines", element_value=",".join([engine for engine in engines])) -for collection in collection_list: - for index in indexes_to_delete_from: - sql = f"delete from {index} where collection='/{source}/{collection}/'" - command_file.update_or_add_element_value(element_name="SQL", element_value=sql, add_duplicate=True) - -file_name = f"{COMMAND_FILES_PATH}{batch_delete_name}.xml" -command_file._update_config_xml(file_name) diff --git a/config_generation/delete_webapp_collections.py b/config_generation/delete_webapp_collections.py deleted file mode 100644 index d50815fb..00000000 --- a/config_generation/delete_webapp_collections.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -this script is used in conjunction with deletes_config_folders.py and delete_content.py to purge a collection -in this case, from the webapp -""" - -from sde_collections.models.collection import Collection - - -def delete_collections_by_config_folder(names_to_delete): - for name in names_to_delete: - Collection.objects.filter(config_folder__exact=name).delete() - print(f"Deleted collections with config folder: {name}") - - -# run this in the shell on your list of collection names -# dmshell is the alias diff --git a/config_generation/export_collections.py b/config_generation/export_collections.py deleted file mode 100644 index d697eb9e..00000000 --- a/config_generation/export_collections.py +++ /dev/null @@ -1,73 +0,0 @@ -import json -import os -import zipfile - -import boto3 -import environ -from api import Api - -# Set the project base directory -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -COLLECTIONS_TO_UPLOAD = [ - # list of collections to upload, eg: "DataPathFinder" -] - -env = environ.Env( - # set casting, default value - DEBUG=(bool, False) -) - -# Take environment variables from .env file -environ.Env.read_env(os.path.join(BASE_DIR, ".env")) - -api = Api("test_server") - -for collection in COLLECTIONS_TO_UPLOAD: - print(f"Running SQL query for collection {collection}...") - response = api.sql("SMD", collection) - - # TODO: save response to a csv with f'{collection}.xml' as the name - candidate_urls = response["Rows"] - bulk_data = [ - { - "url": candidate_url[0], - "scraped_title": candidate_url[1], - } - for candidate_url in candidate_urls - ] - - TEMP_FOLDER_NAME = "temp" - - # Folder to create temporary files - os.makedirs(f"{TEMP_FOLDER_NAME}/{collection}", exist_ok=True) - - # Create JSON file - print("Creating JSON dump...") - json_data = json.dumps(bulk_data) - file_path = f"{TEMP_FOLDER_NAME}/{collection}/urls.json" # Provide the desired file path - with open(file_path, "w") as file: - file.write(json_data) - - # Zip the JSON file - print("Creating zip file...") - zip_file_path = f"{TEMP_FOLDER_NAME}/{collection}.zip" # Provide the desired zip file path - with zipfile.ZipFile(zip_file_path, "w") as zip_file: - zip_file.write(file_path, os.path.basename(file_path)) - - # Upload the zip file to S3 - s3_bucket_name = env("DJANGO_AWS_STORAGE_BUCKET_NAME") - s3_key = f"scraped_urls/{collection}.zip" # Provide the desired S3 key for the uploaded file - s3_client = boto3.client( - "s3", - region_name="us-east-1", - aws_access_key_id=env("DJANGO_AWS_ACCESS_KEY_ID"), - aws_secret_access_key=env("DJANGO_AWS_SECRET_ACCESS_KEY"), - ) - print(f"Uploading to S3 bucket at {s3_key}...") - s3_client.upload_file(zip_file_path, s3_bucket_name, s3_key) - - # Delete the original JSON and zip file - print("Deleting json file and zip file...") - os.remove(file_path) - os.remove(zip_file_path) - print("Success!\n") diff --git a/config_generation/export_whole_index.py b/config_generation/export_whole_index.py deleted file mode 100644 index 3297f679..00000000 --- a/config_generation/export_whole_index.py +++ /dev/null @@ -1,58 +0,0 @@ -import json -import os -import zipfile - -import boto3 -import environ -from api import Api - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -env = environ.Env(DEBUG=(bool, False)) - -environ.Env.read_env(os.path.join(BASE_DIR, ".env")) - -api = Api("test_server") - -response = api.sql("SMD", fetch_all=True) - -candidate_urls = response["Rows"] -bulk_data = [ - { - "url": candidate_url[0], - "scraped_title": candidate_url[1], - "collection": candidate_url[2], - } - for candidate_url in candidate_urls -] - -TEMP_FOLDER_NAME = "temp" - -# Folder to create temporary files -os.makedirs(f"{TEMP_FOLDER_NAME}/all_data", exist_ok=True) - -# Create JSON file -json_data = json.dumps(bulk_data) -file_path = f"{TEMP_FOLDER_NAME}/all_data/urls.json" # Provide the desired file path -with open(file_path, "w") as file: - file.write(json_data) - -# Zip the JSON file -zip_file_path = f"{TEMP_FOLDER_NAME}/all_data.zip" # Provide the desired zip file path -with zipfile.ZipFile(zip_file_path, "w") as zip_file: - zip_file.write(file_path, os.path.basename(file_path)) - -# Upload the zip file to S3 -s3_bucket_name = env("DJANGO_AWS_STORAGE_BUCKET_NAME") -s3_key = "scraped_urls_all/all_data.zip" # Provide the desired S3 key for the uploaded file -s3_client = boto3.client( - "s3", - region_name="us-east-1", - aws_access_key_id=env("DJANGO_AWS_ACCESS_KEY_ID"), - aws_secret_access_key=env("DJANGO_AWS_SECRET_ACCESS_KEY"), -) -s3_client.upload_file(zip_file_path, s3_bucket_name, s3_key) - -# Delete the original JSON and zip file -os.remove(file_path) -os.remove(zip_file_path) diff --git a/config_generation/generate_collection_list.py b/config_generation/generate_collection_list.py deleted file mode 100644 index ee0e9b47..00000000 --- a/config_generation/generate_collection_list.py +++ /dev/null @@ -1,71 +0,0 @@ -"""this document compares creates a list of sources to scrape -- start with turned on sources -- remove already scraped sources -- filter anything that isn't a webcrawler -- provide a variable, turned_on_remaining_webcrawlers for import by other files -""" - -import os - -from db_to_xml import XmlEditor -from sources_to_scrape import ( - already_scraped_sources, - sources_with_documents_20230605, - turned_on_sources, -) - -ROOT_PATH = "../sinequa_configs/sources/SDE/" - - -def create_xml_path(collection_name): - return f"{ROOT_PATH}{collection_name}/default.xml" - - -def get_turned_on_sources(): - # remove sources that were just scraped - turned_on_remaining_sources = [source for source in turned_on_sources if source not in already_scraped_sources] - - # filter all sources to only webcrawler sources - turned_on_remaining_webcrawlers = [] - for collection_name in turned_on_remaining_sources: - path = create_xml_path(collection_name) - indexer = XmlEditor(path) - if "crawler2" in indexer.get_tag_value("Connector"): - turned_on_remaining_webcrawlers.append(collection_name) - - -def is_collection_crawler(collection_name): - config = XmlEditor(create_xml_path(collection_name)) - return "crawler2" in config.get_tag_value("Connector") - - -def get_all_config_folder_names(folder_path=ROOT_PATH): - """ - this returns a list of all the config folder names, not the path of the folder - """ - # Look at each directory directly in the given folder - folders = [] - for dir in os.listdir(folder_path): - dir_path = os.path.join(folder_path, dir) - if os.path.isdir(dir_path) and "default.xml" in os.listdir(dir_path): - # Print the name of the directory - folders.append(dir) - return folders - - -def get_sources_20230605(): - """ - - empty webcrawlers - - 5 sources that were interupted - """ - interrupted_sources = [ - "PDS_Mission_Data_Archive_Website", - "goddard_institute_for_space_studies", - "ASTRO_Image_Cutouts_Website", - "PDS_Mars_Exploration_Program_Website", - ] - folders = get_all_config_folder_names() - folders = folders + interrupted_sources - return [ - folder for folder in folders if folder not in sources_with_documents_20230605 and is_collection_crawler(folder) - ] diff --git a/config_generation/generate_commands.py b/config_generation/generate_commands.py deleted file mode 100644 index 1b41858c..00000000 --- a/config_generation/generate_commands.py +++ /dev/null @@ -1,87 +0,0 @@ -""" -sometimes spot fixes need to be run on a list of collections -this file provides a quick framework to generate a batch of commands based on an input json -""" - -from db_to_xml_file_based import XmlEditor -from generate_jobs import ParallelJobCreator - -from config import source - - -# note that there is an xml folder that contains templates -class CommandGenerator: - def __init__( - self, - command_batch_name, - template_root_path="xmls/", - command_root_path="../sinequa_configs/commands/", - source=source, - ): - self.command_batch_name = command_batch_name # this is used to name the commands - self.template_root_path = template_root_path - self.command_template_path = f"{template_root_path}command_template.xml" - self.job_command_template_path = f"{template_root_path}job_command_template.xml" - self.command_root_path = command_root_path - self.source = source - - def _generate_job_command_name(self, collection_name): - # TODO - # return f"job.{}" - pass - - def _generate_job_command_file_path(self, collection_name): - return f"{self.command_root_path}/{self._generate_command_name(collection_name)}.xml" - - def _generate_command_name(self, collection_name): - """command names are used in the xml file name and are referenced by jobs (without the folder or .xml)""" - return f"{self.command_batch_name}.{collection_name}" - - def _generate_command_file_path(self, collection_name): - return f"{self.command_root_path}/{self._generate_command_name(collection_name)}.xml" - - def generate_command_file(self, collection_name, commands): - command_file = XmlEditor(self.command_template_path) - command_file.update_or_add_element_value( - element_name="WhereClause", - element_value=f"collection='/{self.source}/{collection_name}'", - ) - for command in commands: - command_file.add_column_update( - column=command["Column"], - value=command["Value"], - selection=command.get("Selection", None), - ) - command_file._update_config_xml(self._generate_command_file_path(collection_name)) - - def generate_job_file(self, collection_name): - # each command needs an job file to reference it - job_file = XmlEditor(self.job_command_template_path) - job_file.update_or_add_element_value(element_name="Command", element_value=command_name) - job_file._update_config_xml() - - -# here's how you would list the commands to generate -commands_to_generate = { - "collection_name": [ - { - "Column": "treepath", - "Value": "/Earth Science/Documents/Publications/NTRS Publication Database/", - # 'Selection': 'selection_here' - }, - ], -} - -job_command_names = [] -for collection_name, commands in commands_to_generate.items(): - generator = CommandGenerator(command_batch_name="fix_treeroots") - generator.generate_command_file(collection_name=collection_name, commands=commands) - command_name = generator._generate_command_name(collection_name) - JOB_COMMAND_TEMPLATE_PATH = "config_generation/xmls/job_command_template.xml" - job_file = XmlEditor(JOB_COMMAND_TEMPLATE_PATH) - job_file.update_or_add_element_value(element_name="Command", element_value=command_name) - job_file._update_config_xml() - job_creator = ParallelJobCreator( - [collection_name] - ) # this class is a bit misconfigured for this task, but it works... - job_creator() diff --git a/config_generation/generate_emac_indexer.py b/config_generation/generate_emac_indexer.py deleted file mode 100644 index 4f21c071..00000000 --- a/config_generation/generate_emac_indexer.py +++ /dev/null @@ -1,81 +0,0 @@ -from db_to_xml import XmlEditor - -# collection_metadata -name = "EMAC: Exoplanet Modeling and Analysis Center" -machine_name = "emac_exoplanet_modeling_and_analysis_center" -config_folder = None -url = "https://emac.gsfc.nasa.gov/" -division = "Astrophysics" -update_frequency = "Monthly" -document_type = "Software and Tools" -tree_root = "Astrophysics/Models/EMAC: Exoplanet Modeling and Analysis Center/" - -# rule_metadata -URL_EXCLUDES = [ - "https://emac.gsfc.nasa.gov/?related_resource=*", - "https://emac.gsfc.nasa.gov/news/rss/", -] - -TITLE_RULES = [ - { - "title_criteria": "https://emac.gsfc.nasa.gov/subscriptions/", - "title_value": "EMAC - Tool Category Subscription", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/submissions/", - "title_value": "EMAC - Resource Submission", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/FAQ/", - "title_value": "EMAC - Frequently Asked Questions", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/team/", - "title_value": "EMAC - Team", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/developers/", - "title_value": "EMAC - Software Best Practices and Challenges", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/workshop/", - "title_value": "EMAC - Workshop", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/lightkurve/", - "title_value": "EMAC - WebApp", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/?sort=date", - "title_value": "EMAC - Published Resource List", - }, - { - "title_criteria": "https://emac.gsfc.nasa.gov/?cid=*", - "title_value": """xpath://*[@id="resource_content"]/div[1]/div[1]/div[2]/div""", - }, - # { - # "title_criteria": "https://emac.gsfc.nasa.gov/news/*", - # "title_value": """xpath:/html/body/div[6]/div/div[2]/div/div/h1""", - # }, -] - -# file saving information -ORIGINAL_CONFIG_PATH = "xmls/scraper_template.xml" - -# collection metadata adding -editor = XmlEditor(ORIGINAL_CONFIG_PATH) -editor.convert_scraper_to_indexer() -# editor.add_id() -editor.add_document_type(document_type) -editor.update_or_add_element_value("visibility", "publicCollection") -editor.update_or_add_element_value("Description", f"Webcrawler for the {name}") -editor.update_or_add_element_value("Url", url) -editor.update_or_add_element_value("TreeRoot", tree_root) -editor.update_or_add_element_value("ShardIndexes", "@SMD_ASTRO_Repository_1,@SMD_ASTRO_Repository_2") -editor.update_or_add_element_value("ShardingStrategy", "Balanced") - -# rule adding -[editor.add_url_exclude(url) for url in URL_EXCLUDES] -[editor.add_title_mapping(**title_rule) for title_rule in TITLE_RULES] - -editor.create_config_folder_and_default("indexing_configs", machine_name) diff --git a/config_generation/generate_jobs.py b/config_generation/generate_jobs.py deleted file mode 100644 index f1e8fded..00000000 --- a/config_generation/generate_jobs.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -indexes lots of stuff at once -splits a big list of collections into n subgroups. each collection has a job created. -each indexing job is added to one of the n subgroups. a master runner is made that executes the n -subgroups in parallel -""" - -from db_to_xml_file_based import XmlEditor - -from config import available_indexers, batch_index_name, collection_list, source - - -class ParallelJobCreator: - def __init__( - self, - collection_list, - template_root_path="xmls/", - job_path_root="../sinequa_configs/jobs/", - source=source, - ): - """ - these default values rely on the old file structure, where the sinequa_configs were a - sub-repo of sde-indexing-helper. so when running this, you will need the sde-backend - code to be inside a folder called sinequa_configs - """ - - self.collection_list = collection_list - self.template_root_path = template_root_path - self.joblist_template_path = f"{template_root_path}joblist_template.xml" - self.job_path_root = job_path_root - self.source = source - - def _create_job_name(self, collection_name): - """ - each job that runs an individual collection needs a name based on the collection name - this code generates that file name as a string, and it will be passed to the function that - creates the actual job file - """ - if source == "SDE": - return f"collection.indexer.{collection_name}.xml" - else: - return f"collection.indexer.{source}.{collection_name}.xml" - - def _create_joblist_name(self, index): - """ - each job that runs an list of collections a name based on: - - the date the batch was created - - the index out of n total batches - this code generates that file name as a string, and it will be passed to the function that - creates the actual job file - """ - return f"parallel_indexing_list-{batch_index_name}-{index}.xml" - - def _create_collection_jobs(self): - """ - in order to run a collection, a job must exist that runs it - this code: - - creates a job based on the job template - - adds the exact collection name - - saves it with a name that will reference the collection name - """ - # create single jobs to run each collection - for collection in self.collection_list: - job = XmlEditor(f"{self.template_root_path}job_template.xml") - job.update_or_add_element_value("Collection", f"/{self.source}/{collection}/") - job._update_config_xml(f"{self.job_path_root}{self._create_job_name(collection)}") - - def make_all_parallel_jobs(self): - # create initial single jobs that will be referenced by the parallel job lists - self._create_collection_jobs() - n = len(available_indexers) - # Create an empty list of lists - sublists = [[] for _ in range(n)] - - # Distribute elements of the big list into sublists - for i in range(len(self.collection_list)): - # Use modulus to decide which sublist to put the item in - sublist_index = i % n - sublists[sublist_index].append(self.collection_list[i]) - - # create the n joblists (which will execute their contents serially in parallel - job_names = [] - for index, sublist in enumerate(sublists): - joblist = XmlEditor(self.joblist_template_path) - joblist.update_or_add_element_value("StartIdentity", available_indexers[index]) - for collection in sublist: - joblist.add_job_list_item(self._create_job_name(collection).replace(".xml", "")) - - joblist._update_config_xml(f"{self.job_path_root}{self._create_joblist_name(index)}") - job_names.append(self._create_joblist_name(index).replace(".xml", "")) - - master = XmlEditor(self.joblist_template_path) - master.update_or_add_element_value("RunJobsInParallel", "true") - [master.add_job_list_item(job_name) for job_name in job_names] - master._update_config_xml(f"{self.job_path_root}parallel_indexing_list-{batch_index_name}-master.xml") - - -if __name__ == "__main__": - job_creator = ParallelJobCreator(collection_list=collection_list) - job_creator.make_all_parallel_jobs() diff --git a/config_generation/generate_scrapers.py b/config_generation/generate_scrapers.py deleted file mode 100644 index 8e1b2c7d..00000000 --- a/config_generation/generate_scrapers.py +++ /dev/null @@ -1,66 +0,0 @@ -# this file creates new Candidate URL scraper configs -import os - -from config_utilities import ( - create_config_folder_and_default, - create_folder, - open_xml_as_dict, -) - -# from db_to_xml import convert_indexer_to_scraper - - -def get_or_create_folder(scraper_folder_name="scraping_configs"): - # TODO: probably don't need this function - """ - scraper configs are placed in their own folder above this directory - this gets that folder path - """ - - scraper_folder_path = os.path.join(os.path.dirname(os.getcwd()), scraper_folder_name) - # creates the folder if it doesn't already exist - create_folder(scraper_folder_path) - - return scraper_folder_path - - -def get_scraper_folder(scraper_folder_name="scraping_configs"): - """ - scraper configs are placed in their own folder above this directory - this gets that folder path - """ - - scraper_folder_path = os.path.join(os.path.dirname(os.getcwd()), scraper_folder_name) - # creates the folder if it doesn't already exist - create_folder(scraper_folder_path) - - return scraper_folder_path - - -def generate_brand_new_scraper(source_name, url, scraper_template_path="xmls/scraper_template.xml"): - """ - - Args: - source_name (str): Collection.machine_name - url (str): Collection.url - scraper_template_path (str, optional): path to the template from which to generate the scraper. - Defaults to "xmls/scraper_template.xml". - """ - - scraper = open_xml_as_dict(scraper_template_path) - scraper["Sinequa"]["Url"] = url - - scraper_folder_path = get_scraper_folder() - create_config_folder_and_default(scraper_folder_path, source_name, scraper) - - -# def generate_scraper_based_on_existing(source_name, existing_config_path): -# existing_config_xml = get_tree(existing_config_path) -# scraper_xml = convert_indexer_to_scraper(existing_config_xml) - - -# if __name__ == "__main__": -# from sources_to_scrape import kaylins_new_sources - -# for source in kaylins_new_sources: -# generate_scraper(source["source_name"], source["url"]) diff --git a/config_generation/indexing_configs/emac_exoplanet_modeling_and_analysis_center/default.xml b/config_generation/indexing_configs/emac_exoplanet_modeling_and_analysis_center/default.xml deleted file mode 100644 index 66744d0b..00000000 --- a/config_generation/indexing_configs/emac_exoplanet_modeling_and_analysis_center/default.xml +++ /dev/null @@ -1,364 +0,0 @@ - - - Webcrawler for the EMAC: Exoplanet Modeling and Analysis Center - publicCollection - crawler2 - NodeIndexer1/identity0 - - - - Astrophysics/Models/EMAC: Exoplanet Modeling and Analysis Center/ - false - - - - - - - - - - false - false - false - - - true - - - - - - false - false - false - 0 - - - 20 - - false - - true - false - - - false - false - false - false - true - false - - - - false - false - - false - false - false - - - - - - - - - - - - - - - - false - true - true - false - false - false - - - - false - false - true - false - - _Advanced - true - false - - - - true - - - - - - - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - - - - - - - - - false - - - - - - false - @SMD_ASTRO_Repository_1,@SMD_ASTRO_Repository_2 - Balanced - - - - false - - 8 - - - - true - true - true - 100 - 100000 - 100000 - 10 - -1 - -1 - true - false - false - false - false - false - true - true - false - true - true - true - true - false - 1 - 0 ms - true - no - false - - false - false - True - false - false - - - false - true - true - true - false - true - true - false - false - false - false - false - false - false - - - - true - true - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - true - - - - - - - - - - - false - true - - - - - false - - - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - true - - true - - 80 - true - false - - - - - false - false - - 1 - - https://emac.gsfc.nasa.gov/ - *.rtf;*.jy;*.xml;*.ico;*.gz;*.act - - id - doc.url1 - - - - - - sourcestr56 - - "Software and Tools" - - - - https://emac.gsfc.nasa.gov/?related_resource=* - https://emac.gsfc.nasa.gov/news/rss/ - - title - - "EMAC - Tool Category Subscription" - doc.url1 match "https://emac.gsfc.nasa.gov/subscriptions" - - - - title - - "EMAC - Resource Submission" - doc.url1 match "https://emac.gsfc.nasa.gov/submissions" - - - - title - - "EMAC - Frequently Asked Questions" - doc.url1 match "https://emac.gsfc.nasa.gov/FAQ" - - - - title - - "EMAC - Team" - doc.url1 match "https://emac.gsfc.nasa.gov/team" - - - - title - - "EMAC - Software Best Practices and Challenges" - doc.url1 match "https://emac.gsfc.nasa.gov/developers" - - - - title - - "EMAC - Workshop" - doc.url1 match "https://emac.gsfc.nasa.gov/workshop" - - - - title - - "EMAC - WebApp" - doc.url1 match "https://emac.gsfc.nasa.gov/lightkurve" - - - - title - - "EMAC - Published Resource List" - doc.url1 match "https://emac.gsfc.nasa.gov/?sort=date" - - - - title - - xpath://*[@id="resource_content"]/div[1]/div[1]/div[2]/div - doc.url1 match "https://emac.gsfc.nasa.gov/?cid=*" - - - diff --git a/config_generation/minimum_api.py b/config_generation/minimum_api.py deleted file mode 100644 index ec3cff1e..00000000 --- a/config_generation/minimum_api.py +++ /dev/null @@ -1,81 +0,0 @@ -from typing import Any - -import requests - -server_configs: dict[str, dict[str, str]] = { - "test_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.test.nasa.gov", - }, - "production_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.nasa.gov", - }, -} - - -class Api: - def __init__(self, server_name: str = "test", token: str = None) -> None: - self.app_name: str = server_configs[server_name]["app_name"] - self.query_name: str = server_configs[server_name]["query_name"] - self.base_url: str = server_configs[server_name]["base_url"] - self.token: str = token # you don't need a token for the query endpoint - - def _process_response(self, response) -> dict[str, Any]: - if response.status_code == 200: - meaningful_response = response.json() - else: - meaningful_response = {"response_text": response.text} - - return meaningful_response - - def query(self, term: str, page: int, collection_config_folder=None): - url = f"{self.base_url}/api/v1/search.query" - payload = { - "app": self.app_name, - "query": { - "name": self.query_name, - "action": "search", - "text": term, - "page": page, - "pageSize": 1000, - "tab": "all", - }, - "pretty": "true", - } - - if collection_config_folder: - payload["query"]["collection"] = f"/SMD/{collection_config_folder}/" - - response = requests.post(url, json=payload, verify=False) - - return self._process_response(response) - - def sql(self, source: str = "SMD", collection: str = "", fetch_all: bool = False) -> dict[str, Any]: - if not self.token: - raise ValueError("you must have a token to use the SQL endpoint") - - url = f"{self.base_url}/api/v1/engine.sql" - - collection_name = f"/{source}/{collection}/" - sql_command_all = "select url1,title,collection from @@ScienceMissionDirectorate" - if fetch_all: - sql_command = sql_command_all - else: - sql_command = f"{sql_command_all} where collection='{collection_name}'" - - payload = { - "sql": sql_command, - "maxRows": 10000000, # ten million - "pretty": "true", - } - response = requests.post( - url, - headers={"Authorization": f"Bearer {self.token}"}, - json=payload, - verify=False, - ) - - return self._process_response(response) diff --git a/config_generation/plugins/ListCandidateUrls.cs b/config_generation/plugins/ListCandidateUrls.cs deleted file mode 100644 index c8d2d8f2..00000000 --- a/config_generation/plugins/ListCandidateUrls.cs +++ /dev/null @@ -1,77 +0,0 @@ -/////////////////////////////////////////////////////////// -// Plugin SMD_Plugins : file ListCandidateUrls.cs -// - -using System; -using System.Collections.Generic; -using System.Text; -using System.IO; -using Sinequa.Common; -using Sinequa.Configuration; -using Sinequa.Plugins; -using Sinequa.Connectors; -using Sinequa.Indexer; -using Sinequa.Search; -using System.Text.RegularExpressions; - -namespace Sinequa.Plugin -{ - public class ListCandidateUrls : ConnectorPlugin - { - - List urlObjList = new List(); - string urihost = "" ; - string collection_name; - string seperator = "~?~"; - - - public override Return ApplyMappings(SinequaDoc sdoc, ConnectorDoc doc) - { - Sys.Log("inside ApplyMappings"); - UrlMetadata urlObj = new UrlMetadata(); - urlObj.title = Regex.Replace(doc.GetValue("linktitle"), @"\s+", " ").Trim(); - urlObj.url = doc.GetValue("urioriginalstring").Trim(); - urlObj.treepath = doc.GetValue("urirelpath").Trim(); - urlObjList.Add(urlObj); - urihost = doc.GetValue("uriauthority").Trim(); - - return base.ApplyMappings(sdoc, doc); - } - - public override void OnConnectorEnd() - { - - string root_path = @"C:\sinequa\data\configuration\files\customfile\candidate_urls"; - string collection_name = Connector.CollectionName; - string path = Path.Combine(root_path, collection_name)+".txt"; - - Sys.Log("root_path: ", root_path); - Sys.Log("collection_name: ", collection_name); - Sys.Log("file path: ", path); - // Open a StreamWriter object to write to a text file - using (StreamWriter writer = new StreamWriter(path)) - { - - writer.WriteLine("Website URL~?~Rel folder~?~Title"); - // Iterate through each key-value pair in the dictionary - foreach (var obj in urlObjList) - { - // Write the key and value to the file, separated by a tab character - writer.WriteLine(obj.url + seperator + obj.treepath + seperator + obj.title.Replace(System.Environment.NewLine, "")); - } - writer.Flush(); - writer.Dispose(); - } - } - - - } - - public class UrlMetadata - { - public string title {get;set;} - public string treepath {get;set;} - public string url {get;set;} - } - -} diff --git a/config_generation/preprocess_sources.py b/config_generation/preprocess_sources.py deleted file mode 100644 index 889b94b8..00000000 --- a/config_generation/preprocess_sources.py +++ /dev/null @@ -1,50 +0,0 @@ -from db_to_xml import XmlEditor -from generate_collection_list import create_xml_path, turned_on_remaining_webcrawlers -from sources_to_scrape import remove_top_limitation_sources, turned_on_sources - -ROOT_PATH = "../sinequa_configs/sources/SDE/" - - -def remove_top_only_limitation(path): - # this is only run on the exact sources we want disabled, so checking - # for and retaining previous values is not required - indexer = XmlEditor(path) - indexer.update_or_add_element_value("MaxToIndex", "") - indexer.update_or_add_element_value("MaxLevel", "") - indexer.update_or_add_element_value("MaxToCrawl", "") - indexer._update_config_xml(path) - - -def ensure_index_of_root(path): - # some indexers aren't getting the actual root url given in Url - # this should directly ensure it is scraped - indexer = XmlEditor(path) - urls = indexer.get_tag_value("Url") - for url in urls: - # ensure we don't double add these - if url not in indexer.get_tag_value("UrlIndexIncluded"): - indexer.add_url_include(url) - indexer._update_config_xml(path) - - -# # undo top limitation -# for collection_name in remove_top_limitation_sources: -# path = create_xml_path(collection_name) -# remove_top_only_limitation(path) - -# # ensure root -# for collection_name in turned_on_remaining_webcrawlers: -# path = create_xml_path(collection_name) -# ensure_index_of_root(path) - -# format files... -for collection_name in turned_on_remaining_webcrawlers: - path = create_xml_path(collection_name) - indexer = XmlEditor(path) - indexer._resave_pretty(path) - - -print(len(turned_on_sources)) # 139 -print(len(turned_on_remaining_webcrawlers)) # 114 -print(len(remove_top_limitation_sources)) # 5 -print(len([s for s in remove_top_limitation_sources if s in turned_on_remaining_webcrawlers])) # 5 diff --git a/config_generation/sources_to_scrape.py b/config_generation/sources_to_scrape.py deleted file mode 100644 index 0e0523dc..00000000 --- a/config_generation/sources_to_scrape.py +++ /dev/null @@ -1,1631 +0,0 @@ -# this list is read by generate_scrapers when making a list of sources to scrape -test_sources = [ - {"source_name": "quotes", "url": "https://quotes.toscrape.com"}, -] - -turned_on_sources = [ - "algorithm_theoretical_basis_documents", - "eos_mission_page", - "gcn_circulars", - "gcn_missions_instruments_and_facilities", - "general_coordinates_network_gcn", - "giss_software_tools", - "our_changing_planet_the_view_from_space_images", - "GENELAB_METADATA_Website", - "PDS_EPIC_Model_Website", - "NasaEarthObservationWebsite", - "PDS_PDS4_Training_Documents_Website", - "GENELAB_Github_DataProcessing", - "PDS_Notebook_Website", - "PDS_Metadata_Injector_for_PDS_Labels_Website", - "PDS_Photojournal_Website", - "PDS_ODE_REST_Service_Website", - "PDS_Lunar_Orbiter_Data_Explorer_Website", - "PDS_NASA_Science_Solar_System_Exploration_Website", - "PDS_Messenger_MASCS_UVVS_Archive_Page_Website", - "PDS_Image_Atlas_Website", - "PDS_NASAs_Eyes_Website", - "ASTRO_James_Webb_Space_Telescope_Website", - "SPEDAS_Website", - "CCMC_Website", - "ASTRO_Data_Hosted_on_LAMBDA_Website", - "PDS_Toolkits_Website", - "ASTRO_Hubble_Source_Catalog_Search_API_Website", - "PyHC_Website", - "PDS_Java_Mission-planning_and_Analysis_for_Remote_Sensing_(JMARS)_Website", - "ASTRO_HEASARC_Software_Website", - "PDS_NASA_Science_Earths_Moon_Website", - "Helio_Events_Knowledgebase_Website", - "DataPathFinder", - "NTRS", - "PDS_Juno_Archive_Page_Website", - "PDS_Astropedia_Lunar_and_Planetary_Cartographic_Catalog_Website", - "PDS_Odyssey_GRS_Data_Node_Website", - "SPASE_Website", - "PDS_Astromat_Astromaterials_Data_System_Website", - "PDS_Small_Bodies_Image_Browser_Website", - "PDS_Virtual_Astronaut_Website", - "PDS_Pluto_and_Arrokoth_Data_Archive_Website", - "PDS_Users_Guides_Website", - "PDS_Mars_Orbiter_Data_Website", - "PDS_Mars_Exploration_Program_Website", - "PDS_Mercury_Data_Archive_Website", - "PDS_TES_Data_Node_Website", - "PDS_Jupiter_Data_Archive_Website", - "GENELAB_Github_SampleProcessing", - "PDS_PDS_Annex_Products_Website", - "PDS_PDS_Tool_Registry_Website", - "PDS_SBN_Tools_Utilities_and_Interfaces_Website", - "PDS_ISIS_Astro_Website", - "PDS_Missions_Website", - "GENELAB_Publications_Website", - "PDS_DIVINER_RDR_Query_Website", - "ASTRO_NASA_Exoplanet_Archive_Documents_Website", - "ASTRO_API_Search_Website", - "PDS_High-Resolution_Transmission_Molecular_Absorption_Database_(HITRAN)_Website", - "PDS_Analyst_Notebook_Website", - "ASTRO_ZMAST_API_Website", - "ASTRO_MAST_Documentation_Website", - "PDS_Data_Pilot_Website", - "PDS_PDS_Software_Tools_Tutorial_and_Viewers_Website", - "PDS_Gravity_Models_Website", - "PDS_Venus_Archive_Page_Website", - "Autoplot_Website", - "PDS_Rings_Website", - "PDS_Mission_Data_Archive_Website", - "PDS_LOLA_RDR_Query_Website", - "PDS_PPI_Software_Website", - "PDS_ISIS_Website", - "PDS_USGS_Pilot_Website", - "PDS_Imaging_Software_Website", - "PDS_Missions_Archive_Page_Website", - "PDS_SPICE_Archives_Website", - "PDS_LOLA_RDR_Query_V20_Website", - "PDS_Subscription_Service_Website", - "ASTRO_Calibration_Documentation_Website", - "PDS_Venus_Orbital_Data_Explorer_Website", - "PDS_SPICE_Programming_Lessons_Website", - "PDS_API_Legacy_All", - "ASTRO_Data_Reduction_Tools_Website", - "Helioviewer_Website", - "ASTRO_HIRES_PRV_Website", - "PDS_PDS4_Documents_Website", - "ASTRO_Image_Cutouts_Website", - "PDS_Ring-Moon_Systems_Node_On-line_Tools_Website", - "PDS_Neptune_Archive_Page_Website", - "PDS_Uranus_Data_Archive_Website", - "PDS_Mars_Lander_Data_Website", - "PDS_Lunar_Atmospheres_Data_Archive_Website", - "GENELAB_Github_Training", - "ASTRO_TAP_Search_Website", - "PDS_Mercury_Orbital_Data_Explorer_Website", - "CMR_API", - "PDS_Outer_Planets_Icy_Satellites_Archive_Page_Website", - "ARSETAppliedSciences", - "PDS_Planetary_Science_Tools_Website", - "PDS_SPICE_Utility_and_Application_Programs_Website", - "PDS_Geosciences_Node_Spectral_Library_Website", - "PDS_SPICE_Toolkit_Website", - "PDS_NASA_Solar_System_Treks_Website", - "PDS_PDS_Documentation_Website", - "PDS_Geosciences_Data_Holdings_Website", - "PDS_SPICE-enhanced_Cosmographia_Website", - "ASTRO_Contributed_Datasets_Website", - "PDS_PDS3_Standards_Reference_Website", - "PDS_PDS4_Local_Data_Dictionary_Tool_Website", - "PDS_Saturn_Data_Archive_Website", - "PDS_DIVINER_RDR_Query_V20_Website", - "ASTRO_exoMAST_API_Website", - "ASTRO_Spitzer_Tools_Website", - "PDS_Data_Volumes_Index_Website", - "HAPI_Website", - "PDS_PDS4_JParser_Website", - "PDS_Solar_System_Exploration_Research_Virtual_Institute_(SSERVI)_Website", - "PDS_MRO_Coordinated_Observation_Website", - "PDS_Mars_Orbital_Data_Explorer_Website", - "PDS_Titan_Data_Archive_Website", - "PDS_Atmospheric_Escape_Chemistry_Page_Website", - "PDS_Collision_Induced_Absorption_Model_Website", - "ASTRO_NAVO_HEASARC", - "CASEI_Campaign", - "CASEI_Deployment", - "CASEI_Instrument", - "CASEI_Platform", - "giss_datasets_and_derived_materials", - "giss_publication_list", - "nasa_global_climate_change", - "goddard_institute_for_space_studies", - "earth_observer_publications", - "our_changing_planet_the_view_from_space_images", - "nasa_sea_level_change", - "nasa_carbon_monitoring_system", - "emac_exoplanet_modeling_and_analysis_center", - "algorithm_theoretical_basis_documents", - "emac_exoplanet_modeling_and_analysis_center", - "PDS_SPICE_Tutorials_Website", -] - -kaylins_new_sources = [ - {"source_name": "nasa_power", "url": "https://power.larc.nasa.gov/"}, - { - "source_name": "emac_exoplanet_modeling_and_analysis_center", - "url": "https://emac.gsfc.nasa.gov/", - }, - { - "source_name": "goddard_institute_for_space_studies", - "url": "https://www.giss.nasa.gov/", - }, - { - "source_name": "earth_science_decadal_surveys", - "url": "https://science.nasa.gov/earth-science/decadal-surveys/", - }, - { - "source_name": "exoplanet_opacities_database", - "url": "https://science.data.nasa.gov/opacities/", - }, - { - "source_name": "interactive_multiinstrument_database_of_solar_flares", - "url": "https://data.nas.nasa.gov/helio/portals/solarflares/", - }, - {"source_name": "general_coordinates_network_gcn", "url": "https://gcn.nasa.gov/"}, - { - "source_name": "gcn_missions_instruments_and_facilities", - "url": "https://gcn.nasa.gov/missions", - }, - {"source_name": "gcn_circulars", "url": "https://gcn.nasa.gov/circulars"}, - { - "source_name": "igwn_public_alerts_user_guide", - "url": "https://emfollow.docs.ligo.org/userguide/", - }, - { - "source_name": "algorithm_theoretical_basis_documents", - "url": "https://eospso.nasa.gov/content/algorithm-theoretical-basis-documents", - }, - { - "source_name": "eos_mission_page", - "url": "https://eospso.nasa.gov/content/all-missions", - }, - { - "source_name": "earth_observer_publications", - "url": "https://eospso.nasa.gov/earth-observer-archive/", - }, - { - "source_name": "our_changing_planet_the_view_from_space_images", - "url": "https://eospso.nasa.gov/content/our-changing-planet-view-space", - }, - {"source_name": "nasa_global_climate_change", "url": "https://climate.nasa.gov/"}, - {"source_name": "giss_publication_list", "url": "https://pubs.giss.nasa.gov/"}, - {"source_name": "giss_software_tools", "url": "https://www.giss.nasa.gov/tools/"}, - { - "source_name": "giss_datasets_and_derived_materials", - "url": "https://data.giss.nasa.gov/", - }, - { - "source_name": "nasa_wavelength", - "url": "https://science.nasa.gov/learners/wavelength", - }, - {"source_name": "my_nasa_data", "url": "https://mynasadata.larc.nasa.gov/"}, - { - "source_name": "mars_target_encyclopeida_mte", - "url": "https://github.com/wkiri/MTE", - }, - { - "source_name": "astrogeology_analysis_ready_data", - "url": "https://stac.astrogeology.usgs.gov/docs/", - }, - { - "source_name": "nasa_science_missions_earth", - "url": "https://science.nasa.gov/missions-page?field_division_tid=103&field_phase_tid=All", - }, - {"source_name": "f_prime", "url": "https://github.com/nasa/fprime"}, - {"source_name": "nasa_sea_level_change", "url": "https://sealevel.nasa.gov/"}, - {"source_name": "earth_observing_dashboard", "url": "https://eodashboard.org/"}, - { - "source_name": "fire_information_for_resource_management_system_firms", - "url": "https://firms.modaps.eosdis.nasa.gov/", - }, - {"source_name": "nasa_carbon_monitoring_system", "url": "https://carbon.nasa.gov/"}, - { - "source_name": "nasa_2023_climate_strategy", - "url": "https://www.nasa.gov/sites/default/files/atoms/files/advancing_nasas_climate_strategy_2023.pdf", - }, -] -# i've started running up through igwn -sprint_2_sources = [ - { - "source_name": "nasa_science_solar_system_exploration", - "url": "https://solarsystem.nasa.gov/", - }, - { - "source_name": "emac_exoplanet_modeling_and_analysis_center", - "url": "https://emac.gsfc.nasa.gov/", - }, - {"source_name": "mars_exploration_program", "url": "https://mars.nasa.gov/"}, - { - "source_name": "astropedia_lunar_and_planetary_cartographic_catalog", - "url": "https://astrogeology.usgs.gov/search?pmi-target=mercury", - }, - { - "source_name": "pds_cassini_resource_page_website", - "url": "https://pds-atmospheres.nmsu.edu/data_and_services/atmospheres_data/Cassini/Cassini.html", - }, - { - "source_name": "goddard_institute_for_space_studies", - "url": "https://www.giss.nasa.gov/", - }, - { - "source_name": "missions_archive_page", - "url": "https://pds-ppi.igpp.ucla.edu/mission", - }, - { - "source_name": "pds_pds_small_bodies_node_asteroid_dust_subnode_website", - "url": "https://sbn.psi.edu/pds/", - }, - { - "source_name": "earth_science_decadal_surveys", - "url": "https://science.nasa.gov/earth-science/decadal-surveys/", - }, - { - "source_name": "exoplanet_opacities_database", - "url": "https://science.data.nasa.gov/opacities/", - }, - {"source_name": "nasa_power", "url": "https://power.larc.nasa.gov/"}, - { - "source_name": "our_changing_planet_the_view_from_space_images", - "url": "https://eospso.nasa.gov/content/our-changing-planet-view-space", - }, - { - "source_name": "igwn_public_alerts_user_guide", - "url": "https://emfollow.docs.ligo.org/userguide/", - }, - {"source_name": "nasa_global_climate_change", "url": "https://climate.nasa.gov/"}, - { - "source_name": "recently_archived_volumes", - "url": "https://pds-atmospheres.nmsu.edu/data_and_services/atmospheres_data/recent.htm", - }, - {"source_name": "lsda_website", "url": "https://nlsp.nasa.gov/explore/lsdahome"}, - { - "source_name": "small_bodies_data_ferret", - "url": "https://sbnapps.psi.edu/ferret/listDatasets.action", - }, - { - "source_name": "pds_data_archive_website", - "url": "https://pds-imaging.jpl.nasa.gov/data/", - }, - { - "source_name": "interactive_multiinstrument_database_of_solar_flares", - "url": "https://data.nas.nasa.gov/helio/portals/solarflares/", - }, - {"source_name": "general_coordinates_network_gcn", "url": "https://gcn.nasa.gov/"}, - { - "source_name": "gcn_missions_instruments_and_facilities", - "url": "https://gcn.nasa.gov/missions", - }, - { - "source_name": "heliophysics_events_knowledgebase", - "url": "https://www.lmsal.com/hek/", - }, - { - "source_name": "planetary_plasma_interactions_data_volumes", - "url": "https://pds-ppi.igpp.ucla.edu/search/?s=*", - }, - { - "source_name": "astromaterials_acquisition_and_curation_office", - "url": "https://curator.jsc.nasa.gov/", - }, - { - "source_name": "nasa_wavelength", - "url": "https://science.nasa.gov/learners/wavelength", - }, - {"source_name": "nasa_sea_level_change", "url": "https://sealevel.nasa.gov/"}, -] - -finished_sources = [ - "algorithm_theoretical_basis_documents", - "astrogeology_analysis_ready_data", - "astromaterials_acquisition_and_curation_office", - "astropedia_lunar_and_planetary_cartographic_catalog", - "earth_observing_dashboard", - "earth_science_decadal_surveys", - "emac_exoplanet_modeling_and_analysis_center", - "eos_mission_page", - "exoplanet_opacities_database", - "fire_information_for_resource_management_system_firms", - "f_prime", - "gcn_circulars", - "gcn_missions_instruments_and_facilities", - "general_coordinates_network_gcn", - "giss_datasets_and_derived_materials", - "giss_publication_list", - "giss_software_tools", - "goddard_institute_for_space_studies", - "heliophysics_events_knowledgebase", - "igwn_public_alerts_user_guide", - "nasa_science_earths_moon", - "pds4_documents", - "pds_near_earth_asteroid_rendezvous_near_data_archive_website", - "solar_system_exploration_research_virtual_institute_sservi", - "SPASE_JSON_List", -] - -sprint_3_sources = [ - { - "source_name": "solar_system_exploration_research_virtual_institute_sservi", - "url": "https://sservi.nasa.gov/", - }, - { - "source_name": "mars_gcm", - "url": "https://pds-atmospheres.nmsu.edu/PDS/data/mogc_0001", - }, - {"source_name": "pds_archive_navigator_website", "url": "https://arcnav.psi.edu/"}, - { - "source_name": "pds_near_earth_asteroid_rendezvous_near_data_archive_website", - "url": "https://arcnav.psi.edu/urn:nasa:pds:context:investigation:mission.near_earth_asteroid_rendezvous", - }, - {"source_name": "my_nasa_data", "url": "https://mynasadata.larc.nasa.gov/"}, - {"source_name": "mars_target_encyclopedia", "url": "https://github.com/wkiri/MTE"}, - { - "source_name": "astrogeology_analysis_ready_data", - "url": "https://stac.astrogeology.usgs.gov/docs/", - }, - {"source_name": "heasarc", "url": "https://heasarc.gsfc.nasa.gov/"}, - {"source_name": "nasa_science_earths_moon", "url": "https://moon.nasa.gov/"}, - { - "source_name": "pds4_documents", - "url": "https://pds.nasa.gov/datastandards/documents/", - }, - {"source_name": "code_nasa_api", "url": "https://impact.earthdata.nasa.gov/casei/"}, - {"source_name": "nasa_earth_observations", "url": "https://neo.gsfc.nasa.gov/"}, - { - "source_name": "nasa_science_missions_earth", - "url": "https://science.nasa.gov/missions-page?field_division_tid=103&field_phase_tid=All", - }, - {"source_name": "f_prime", "url": "https://github.com/nasa/fprime"}, - {"source_name": "ceos_missions", "url": "https://mims.nasa-impact.net/docs"}, - { - "source_name": "caldb_documentation", - "url": "https://heasarc.gsfc.nasa.gov/docs/heasarc/caldb/caldb_doc.html", - }, - { - "source_name": "genelab_metadata_api", - "url": "https://genelab.nasa.gov/genelabAPIs#metadata", - }, - { - "source_name": "generic_kernels", - "url": "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/", - }, - {"source_name": "casei", "url": "https://impact.earthdata.nasa.gov/casei/"}, - {"source_name": "errata", "url": "https://pds-ppi.igpp.ucla.edu/search/?e="}, - {"source_name": "ceos_instruments", "url": "https://mims.nasa-impact.net/docs"}, - {"source_name": "james_webb_space_telescope", "url": "https://webb.nasa.gov/"}, - { - "source_name": "casei_instruments", - "url": "https://admg.nasa-impact.net/api/docs/", - }, - { - "source_name": "highenergy_missions", - "url": "https://heasarc.gsfc.nasa.gov/docs/heasarc/missions/alphabet.html", - }, - { - "source_name": "arset_applied_sciences", - "url": "https://appliedsciences.nasa.gov/join-mission/training", - }, - { - "source_name": "casei_deployments", - "url": "https://admg.nasa-impact.net/api/docs/", - }, - { - "source_name": "operational_flightother_project_kernels", - "url": "https://naif.jpl.nasa.gov/naif/data_operational.html", - }, - { - "source_name": "spice_programming_lessons", - "url": "https://naif.jpl.nasa.gov/naif/lessons.html", - }, - { - "source_name": "spice_selftraining", - "url": "https://naif.jpl.nasa.gov/naif/self_training.html", - }, - { - "source_name": "spice_toolkit_documentation", - "url": "https://naif.jpl.nasa.gov/naif/documentation.html", - }, - { - "source_name": "spice_tutorials", - "url": "https://naif.jpl.nasa.gov/naif/tutorials.html", - }, - { - "source_name": "mast_documentation", - "url": "https://outerspace.stsci.edu/display/MASTDOCS/Portal+Guidel", - }, - {"source_name": "ccmc", "url": "https://ccmc.gsfc.nasa.gov/"}, - {"source_name": "nasa_carbon_monitoring_system", "url": "https://carbon.nasa.gov/"}, - { - "source_name": "nasa_2023_climate_strategy", - "url": "https://www.nasa.gov/sites/default/files/atoms/files/advancing_nasas_climate_strategy_2023.pdf", - }, - {"source_name": "nasa_sea_level_change", "url": "https://sealevel.nasa.gov/"}, - {"source_name": "earth_observing_dashboard", "url": "https://eodashboard.org/"}, - {"source_name": "earth_observing_dashboard", "url": "https://eodashboard.org/"}, - { - "source_name": "fire_information_for_resource_management_system_firms", - "url": "https://firms.modaps.eosdis.nasa.gov/", - }, -] - -# scraped on test server with the new title code -# TODO: are all of these actually showing? -already_scraped_sources = [ - "ASTRO_Missions_and_Data_Website", - "CODE_NASA_API", - "NasaEarthObservationWebsite", - "DataPathFinder", - "ASTRO_James_Webb_Space_Telescope_Website", - "ASTRO_High-Energy_Missions_Website", - "ARSETAppliedSciences", - "GENELAB_Publications_Website", -] - -# emily put together a list of sources which were previously limited but we now -# want more than the top page: -# https://docs.google.com/spreadsheets/d/1U95xEseibgoHmTSyT6akPxuMVRIUeEu2vOyu9i8teb4/edit#gid=0 -# however, many of these need special attention, so this list in this variable isn't complete -remove_top_limitation_sources = [ - "Autoplot_Website", - "PyHC_Website", - "SPEDAS_Website", - "ASTRO_Hubble_Source_Catalog_Search_API_Website", - "PDS_PDS_Tool_Registry_Website", -] - -lis_new_sources = [ - "algorithm_theoretical_basis_documents", - "eos_mission_page", - "gcn_circulars", - "gcn_missions_instruments_and_facilities", - "general_coordinates_network_gcn", - "giss_software_tools", - "our_changing_planet_the_view_from_space_images", -] - -all_sources = kaylins_new_sources + sprint_2_sources + sprint_3_sources -remaining_sources = [s for s in all_sources if s["source_name"] not in finished_sources] - - -sources_with_documents_20230605 = [ - "algorithm_theoretical_basis_documents", - "ARSETAppliedSciences", - "ASTRO_API_Search_Website", - "ASTRO_Calibration_Documentation_Website", - "ASTRO_Contributed_Datasets_Website", - "ASTRO_Data_Reduction_Tools_Website", - "ASTRO_exoMAST_API_Website", - "ASTRO_HEASARC_Software_Website", - "ASTRO_HIRES_PRV_Website", - "ASTRO_Hubble_Source_Catalog_Search_API_Website", - "ASTRO_Image_Cutouts_Website", - "ASTRO_James_Webb_Space_Telescope_Website", - "ASTRO_MAST_Documentation_Website", - "ASTRO_NASA_Exoplanet_Archive_Documents_Website", - "ASTRO_NAVO_HEASARC", - "ASTRO_Spitzer_Tools_Website", - "ASTRO_TAP_Search_Website", - "ASTRO_ZMAST_API_Website", - "astrogeology_analysis_ready_data", - "CASEI_Campaign", - "CASEI_Deployment", - "CASEI_Instrument", - "CASEI_Platform", - "CMR_API", - "DataPathFinder", - "earth_observer_publications", - "earth_observing_dashboard", - "earth_science_decadal_surveys", - "eos_mission_page", - "f_prime", - "fire_information_for_resource_management_system_firms", - "gcn_circulars", - "gcn_missions_instruments_and_facilities", - "GENELAB_Github_DataProcessing", - "GENELAB_Github_SampleProcessing", - "GENELAB_Github_Training", - "GENELAB_METADATA_Website", - "GENELAB_Publications_Website", - "general_coordinates_network_gcn", - "giss_datasets_and_derived_materials", - "giss_publication_list", - "giss_software_tools", - "goddard_institute_for_space_studies", - "interactive_multiinstrument_database_of_solar_flares", - "mars_target_encyclopedia_mte", - "my_nasa_data", - "nasa_2023_climate_strategy", - "nasa_carbon_monitoring_system", - "nasa_global_climate_change", - "nasa_sea_level_change", - "NasaEarthObservationWebsite", - "our_changing_planet_the_view_from_space_images", - "PDS_Analyst_Notebook_Website", - "PDS_API_Legacy_All", - "PDS_Astromat_Astromaterials_Data_System_Website", - "PDS_Astropedia_Lunar_and_Planetary_Cartographic_Catalog_Website", - "PDS_Atmospheric_Escape_Chemistry_Page_Website", - "PDS_Collision_Induced_Absorption_Model_Website", - "PDS_Data_Pilot_Website", - "PDS_Data_Volumes_Index_Website", - "PDS_DIVINER_RDR_Query_V20_Website", - "PDS_DIVINER_RDR_Query_Website", - "PDS_EPIC_Model_Website", - "PDS_Geosciences_Data_Holdings_Website", - "PDS_Geosciences_Node_Spectral_Library_Website", - "PDS_Gravity_Models_Website", - "PDS_High-Resolution_Transmission_Molecular_Absorption_Database_(HITRAN)_Website", - "PDS_Image_Atlas_Website", - "PDS_Imaging_Software_Website", - "PDS_ISIS_Astro_Website", - "PDS_ISIS_Website", - "PDS_Java_Mission-planning_and_Analysis_for_Remote_Sensing_(JMARS)_Website", - "PDS_Juno_Archive_Page_Website", - "PDS_Jupiter_Data_Archive_Website", - "PDS_LOLA_RDR_Query_V20_Website", - "PDS_LOLA_RDR_Query_Website", - "PDS_Lunar_Atmospheres_Data_Archive_Website", - "PDS_Lunar_Orbiter_Data_Explorer_Website", - "PDS_Mars_Exploration_Program_Website", - "PDS_Mars_Lander_Data_Website", - "PDS_Mars_Orbital_Data_Explorer_Website", - "PDS_Mars_Orbiter_Data_Website", - "PDS_Mercury_Data_Archive_Website", - "PDS_Mercury_Orbital_Data_Explorer_Website", - "PDS_Messenger_MASCS_UVVS_Archive_Page_Website", - "PDS_Metadata_Injector_for_PDS_Labels_Website", - "PDS_Missions_Archive_Page_Website", - "PDS_MRO_Coordinated_Observation_Website", - "PDS_NASA_Science_Earths_Moon_Website", - "PDS_NASA_Science_Solar_System_Exploration_Website", - "PDS_NASAs_Eyes_Website", - "PDS_Neptune_Archive_Page_Website", - "PDS_Notebook_Website", - "PDS_Operational_Flight_Other_Project_Kernels_Website", - "PDS_Outer_Planets_Icy_Satellites_Archive_Page_Website", - "PDS_PDS3_Standards_Reference_Website", - "PDS_PDS4_Documents_Website", - "PDS_PDS4_JParser_Website", - "PDS_PDS4_Local_Data_Dictionary_Tool_Website", - "PDS_PDS4_Training_Documents_Website", - "PDS_PDS_Annex_Products_Website", - "PDS_PDS_Documentation_Website", - "PDS_PDS_Software_Tools_Tutorial_and_Viewers_Website", - "PDS_PDS_Tool_Registry_Website", - "PDS_Photojournal_Website", - "PDS_Planetary_Science_Tools_Website", - "PDS_Pluto_and_Arrokoth_Data_Archive_Website", - "PDS_PPI_Software_Website", - "PDS_Ring-Moon_Systems_Node_On-line_Tools_Website", - "PDS_Rings_Website", - "PDS_Saturn_Data_Archive_Website", - "PDS_SBN_Tools_Utilities_and_Interfaces_Website", - "PDS_Small_Bodies_Image_Browser_Website", - "PDS_Solar_System_Exploration_Research_Virtual_Institute_(SSERVI)_Website", - "PDS_SPICE-enhanced_Cosmographia_Website", - "PDS_SPICE_Archives_Website", - "PDS_SPICE_Programming_Lessons_Website", - "PDS_SPICE_Toolkit_Website", - "PDS_SPICE_Tutorials_Website", - "PDS_SPICE_Utility_and_Application_Programs_Website", - "PDS_Subscription_Service_Website", - "PDS_TES_Data_Node_Website", - "PDS_Titan_Data_Archive_Website", - "PDS_Toolkits_Website", - "PDS_Uranus_Data_Archive_Website", - "PDS_Users_Guides_Website", - "PDS_USGS_Pilot_Website", - "PDS_Venus_Archive_Page_Website", - "PDS_Venus_Orbital_Data_Explorer_Website", - "PDS_Virtual_Astronaut_Website", -] - - -created_from_scratch = [ - "archived_synthetic_data", - "astroquery_api_search_mast_queries", - "co_plotter", - "contributed_datasets_in_the_exoplanet_archive", - "coordinate_calculator", - "corot_exoplanet_archive_etss_data_sets", - "exo_mast", - "exoplanet_atmosphere_observability_table", - "extinction_calculator", - "heasarc_browse_batch_interface", - "heasarc_download_scripts", - "high_level_science_products", - "hubble_source_catalog_search", - "ipac_table_validator", - "koa_program_friendly_image_access_service", - "lbti_archive", - "mast", - "mast_api_search", - "mast_hubble_search", - "mast_portal", - "mast_query_casjobs", - "mast_web_services", - "montage_mosaic_engine", - "neid_archive", - "neid_solar_radial_velocity_archive", - "nexsci", - "pan_starrs_catalog", - "pan_starrs_catalog_api", - "planetary_image_galleries", - "pykoa", - "skiff_spectral_catalog_search", - "space_telescope_bibliographic_search", - "spectral_classes_of_like_stars", - "vao_datascope", - "velocity_calculator", - "venus_data_archive", - "virtual_observatory_information", - "z_mast_search", - "ASTRO_HEASARC_Tools_Website", - "ASTRO_HIRES_PRV_Website", - "ASTRO_Image_Cutouts_Website", - "ASTRO_Spitzer_Tools_Website", - "CMR_API", - "PDS_Annex_Data_Holdings_Website", - "PDS_CRISM_Analysis_Toolkit_(CAT)_Website", - "PDS_Cassini_Mission_Rhea_(Saturn_V)_Website", - "PDS_Cassini_Mission_Saturn_Small_Satellites_Website", - "PDS_Data_Dictionary_Search", - "PDS_MRO_Coordinated_Observation_Website", - "PDS_Mars_Exploration_Program_Website", - "PDS_Mars_Orbital_Data_Explorer_Website", - "PDS_Mission_Data_Archive_Website", - "PDS_Missions_Archive_Page_Website", - "PDS_NASA_Space_Science_Data_Coordinated_Archive_(NSSDCA)_Website", - "PDS_SPICE_Self-Training_Website", - "archived_synthetic_data", - "astroquery_api_search_mast_queries", - "co_plotter", - "contributed_datasets_in_the_exoplanet_archive", - "coordinate_calculator", - "corot_exoplanet_archive_etss_data_sets", - "earth_observer_publications", - "earth_observing_dashboard", - "exo_mast", - "exoplanet_atmosphere_observability_table", - "extinction_calculator", - "f_prime", - "goddard_institute_for_space_studies", - "heasarc_browse_batch_interface", - "heasarc_download_scripts", - "high_level_science_products", - "hubble_source_catalog_search", - "ipac_table_validator", - "koa_program_friendly_image_access_service", - "lbti_archive", - "mast", - "mast_api_search", - "mast_hubble_search", - "mast_portal", - "mast_query_casjobs", - "mast_web_services", - "montage_mosaic_engine", - "neid_archive", - "neid_solar_radial_velocity_archive", - "nexsci", - "pan_starrs_catalog", - "pan_starrs_catalog_api", - "planetary_image_galleries", - "pykoa", - "skiff_spectral_catalog_search", - "space_telescope_bibliographic_search", - "spectral_classes_of_like_stars", - "vao_datascope", - "velocity_calculator", - "venus_data_archive", - "virtual_observatory_information", - "z_mast_search", -] - - -# these are all the ones changed on 2023.08.17 -# update this note after running them -sources_with_new_source56_mappings = [ - "CASEI_Deployment", - "CASEI_Instrument", - "CASEI_Platform", - "CEOS_API_I", - "earth_observing_dashboard", - "f_prime", - "nasa_carbon_monitoring_system", - "nasa_global_climate_change", - "nasa_power", - "nasa_science_missions_earth", - "nasa_sea_level_change", - "nasa_wavelength", -] - -# these are all the ones changed on 2023.08.17 -# update this note after running them -sources_with_new_selection_rule = [ - "PDS_Mars_Exploration_Program_Website", - "goddard_institute_for_space_studies", - "my_nasa_data", -] - -# these are all the ones changed on 2023.08.17 -# update this note after running them -sources_with_new_title_rule = [ - "ASTRO_Calibration_Documentation_Website", - "ASTRO_Missions_and_Data_Website", - "PDS_Astromaterials_Acquisition_and_Curation_Office_Website", - "PDS_Cassini_Mission_Saturn_Small_Satellites_Website", - "gcn_missions_instruments_and_facilities", - "general_coordinates_network_gcn", - "nasa_power", -] - -# this is every single source on the dev branch -# we are burning the test server and rescraping all of these... -sources_to_scrape_20230929 = [ - "ARSETAppliedSciences", - "ASTRO_API_Search_Website", - "ASTRO_Astrophysics_Documents_Website", - "ASTRO_Calibration_Documentation_Website", - "ASTRO_Contributed_Datasets_Website", - "ASTRO_Data_Hosted_on_LAMBDA_Website", - "ASTRO_Data_Reduction_Tools_Website", - "ASTRO_Exoplanet_Program_Documents_Website", - "ASTRO_Finder_Chart_Website", - "ASTRO_HEASARC_Software_Website", - "ASTRO_HEASARC_Tools_Website", - "ASTRO_HIRES_PRV_Website", - "ASTRO_High-Energy_Missions_Website", - "ASTRO_Hubble_Source_Catalog_Search_API_Website", - "ASTRO_Image_Cutouts_Website", - "ASTRO_James_Webb_Space_Telescope_Website", - "ASTRO_MAST_Documentation_Website", - "ASTRO_Missions_and_Data_Website", - "ASTRO_Multi_Website", - "ASTRO_NASA_Exoplanet_Archive_Documents_Website", - "ASTRO_NAVO_HEASARC", - "ASTRO_NED_User_Guides_Website", - "ASTRO_Planck_Cutout_Visualization_Website", - "ASTRO_Spitzer_Tools_Website", - "ASTRO_TAP_Search_Website", - "ASTRO_WISE_Image_Service_Website", - "ASTRO_ZMAST_API_Website", - "ASTRO_exoMAST_API_Website", - "Algorithm_Publication_Tool", - "Autoplot_Website", - "CASEI", - "CASEI_Campaign", - "CASEI_Deployment", - "CASEI_Instrument", - "CASEI_Platform", - "CCMC_Website", - "CEOS_API_I", - "CEOS_API_M", - "CMR_API", - "CODE_NASA_API", - "Commercial_Smallsat_Data_Acquisition_CSDA_Program", - "DataPathFinder", - "ESSCOR_API", - "GCIS_ARTICLE_API", - "GCIS_BOOKS_API", - "GCIS_JOURNAL_API", - "GCIS_REPORTS_API", - "GENELAB_Github_DataProcessing", - "GENELAB_Github_SampleProcessing", - "GENELAB_Github_Training", - "GENELAB_METADATA_Website", - "GENELAB_Publications_Website", - "HAPI_API", - "HAPI_Website", - "Helio_Events_Knowledgebase_Website", - "Heliophysics_Project_Data_Management_Plans", - "Helioviewer_Documentation_Website", - "Helioviewer_Website", - "LSDA_Website", - "LSDA_Website_Trial", - "LSDA_Website_Trial2", - "NASA_Black_Marble", - "NASA_Climate_Change", - "NASA_Earth_Observations", - "NASA_Earth_Observatory", - "NASA_Earthdata", - "NASA_Heliophysics_Digital_Resource_Library_HDRL", - "NASA_SPoRT", - "NASA_Techport", - "NASA_Worldview", - "NAVO_HEASARC", - "NTRS", - "NasaEarthObservationWebsite", - "Optical_Constants_Database", - "PDS_API_Legacy_All", - "PDS_All_Data_Holdings_Website", - "PDS_Analyst_Notebook_Website", - "PDS_Annex_Data_Holdings_Website", - "PDS_Archive_Navigator_Website", - "PDS_Astrogeology_Website", - "PDS_Astromat_Astromaterials_Data_System_Website", - "PDS_Astromaterials_Acquisition_and_Curation_Office_Website", - "PDS_Astropedia_Lunar_and_Planetary_Cartographic_Catalog_Website", - "PDS_Atmospheric_Escape_Chemistry_Page_Website", - "PDS_CRISM_Analysis_Toolkit_(CAT)_Website", - "PDS_CRISM_Spectral_Library_Website", - "PDS_Cassini_Mission_Dione_(Saturn_IV)_Website", - "PDS_Cassini_Mission_Enceladus_(Saturn_II)_Website", - "PDS_Cassini_Mission_Iapetus_(Saturn_VIII)_Website", - "PDS_Cassini_Mission_Mimas_(Saturn_I)_Website", - "PDS_Cassini_Mission_Rhea_(Saturn_V)_Website", - "PDS_Cassini_Mission_Saturn_Small_Satellites_Website", - "PDS_Cassini_Mission_Tethys_(Saturn_III)_Website", - "PDS_Cassini_Resource_Page_Website", - "PDS_Collision_Induced_Absorption_Model_Website", - "PDS_Current_Missions_Website", - "PDS_DIVINER_RDR_Query_V20_Website", - "PDS_DIVINER_RDR_Query_Website", - "PDS_Data_Archive_Website", - "PDS_Data_Dictionary_Search", - "PDS_Data_Pilot_Website", - "PDS_Data_Portal_Website", - "PDS_Data_Volumes_Index_Website", - "PDS_Data_Volumes_Website", - "PDS_Dawn_Mission_to_Ceres_Website", - "PDS_Dawn_Mission_to_Vesta_Website", - "PDS_EPIC_Model_Website", - "PDS_Errata_Website", - "PDS_Generic_Kernels_Website", - "PDS_Geosciences_Data_Holdings_Website", - "PDS_Geosciences_Node_Spectral_Library_Website", - "PDS_Gravity_Models_Website", - "PDS_High-Resolution_Transmission_Molecular_Absorption_Database_(HITRAN)_Website", - "PDS_ISIS_Astro_Website", - "PDS_ISIS_Website", - "PDS_Image_Atlas_Website", - "PDS_Imaging_Software_Website", - "PDS_Java_Mission-planning_and_Analysis_for_Remote_Sensing_(JMARS)_Website", - "PDS_Juno_Archive_Page_Website", - "PDS_Jupiter_Data_Archive_Website", - "PDS_LADEE_NMS_Calibrated_Data_Search", - "PDS_LADEE_NMS_Derived_Data_Search", - "PDS_LADEE_UVS_Calibrated_Data_Search", - "PDS_LOLA_RDR_Query_V20_Website", - "PDS_LOLA_RDR_Query_Website", - "PDS_Lunar_Atmospheres_Data_Archive_Website", - "PDS_Lunar_Orbiter_Data_Explorer_Website", - "PDS_MAVEN_ACC_Data_Search", - "PDS_MAVEN_NGIMS_Data_Search", - "PDS_MRO_Coordinated_Observation_Website", - "PDS_Map-a-Planet_(MAP)_Website", - "PDS_Mars_Exploration_Program_Website", - "PDS_Mars_GCM_Website", - "PDS_Mars_Lander_Data_Website", - "PDS_Mars_Orbital_Data_Explorer_Website", - "PDS_Mars_Orbiter_Data_Website", - "PDS_Mercury_Data_Archive_Website", - "PDS_Mercury_Orbital_Data_Explorer_Website", - "PDS_Messenger_MASCS_UVVS_Archive_Page_Website", - "PDS_Metadata_Injector_for_PDS_Labels_Website", - "PDS_Mission_Data_Archive_Website", - "PDS_Missions_Archive_Page_Website", - "PDS_Missions_Website", - "PDS_Models_and_Simulations_Website", - "PDS_NASA_Science_Earths_Moon_Website", - "PDS_NASA_Science_Solar_System_Exploration_Website", - "PDS_NASA_Solar_System_Treks_Website", - "PDS_NASA_Space_Science_Data_Coordinated_Archive_(NSSDCA)_Website", - "PDS_NASAs_Eyes_Website", - "PDS_NEAR_Shoemaker_Mission_to_433_Eros_Website", - "PDS_Near_Earth_Asteroid_Rendezvous_(NEAR)_Data_Archive_Website", - "PDS_Neptune_Archive_Page_Website", - "PDS_New_Horizons_Encounter_with_Pluto_Website", - "PDS_Niels_Bohr_Institute_Website", - "PDS_Notebook_Website", - "PDS_ODE_REST_Service_Website", - "PDS_OMEGA_Analysis_Toolkit_(OAT)_Website", - "PDS_OPUS_Website", - "PDS_OSIRIS-REx_Mission_to_Bennu_Website", - "PDS_Object_Access_Library_Website", - "PDS_Odyssey_GRS_Data_Node_Website", - "PDS_Operational_Flight_Other_Project_Kernels_Website", - "PDS_Outer_Planets_Icy_Satellites_Archive_Page_Website", - "PDS_PDS3_Standards_Reference_Website", - "PDS_PDS4_Documents_Website", - "PDS_PDS4_JParser_Website", - "PDS_PDS4_Local_Data_Dictionary_Tool_Website", - "PDS_PDS4_Training_Documents_Website", - "PDS_PDS_Annex_Products_Website", - "PDS_PDS_Atmospheres_Data_Set_Catalog_Website", - "PDS_PDS_Documentation_Website", - "PDS_PDS_Small_Bodies_Node_Asteroid_Dust_Subnode_Website", - "PDS_PDS_Software_Tools_Tutorial_and_Viewers_Website", - "PDS_PDS_Tool_Registry_Website", - "PDS_PPI_Documents_Website", - "PDS_PPI_Software_Website", - "PDS_Phoebe_Saturn_IX_Website", - "PDS_Photojournal_Website", - "PDS_Planetary_Data_System_(PDS)_Website", - "PDS_Planetary_Science_Tools_Website", - "PDS_Pluto_and_Arrokoth_Data_Archive_Website", - "PDS_Projection_on_the_Web_(POW)_Service_Website", - "PDS_Recently_Archived_Volumes_Website", - "PDS_Ring-Moon_Systems_Node_On-line_Tools_Website", - "PDS_Rings_Website", - "PDS_SBIB_3D_Website", - "PDS_SBN_Tools_Utilities_and_Interfaces_Website", - "PDS_SPICE-enhanced_Cosmographia_Website", - "PDS_SPICE_Archives_Website", - "PDS_SPICE_Programming_Lessons_Website", - "PDS_SPICE_Self-Training_Website", - "PDS_SPICE_Toolkit_Documentation_Website", - "PDS_SPICE_Toolkit_Website", - "PDS_SPICE_Tutorials_Website", - "PDS_SPICE_Utility_and_Application_Programs_Website", - "PDS_Saturn_Data_Archive_Website", - "PDS_Small_Bodies_Data_Ferret_Website", - "PDS_Small_Bodies_Image_Browser_Website", - "PDS_Solar_System_Exploration_Research_Virtual_Institute_(SSERVI)_Website", - "PDS_Subscription_Service_Website", - "PDS_Subset_Tool_Website", - "PDS_TES_Data_Node_Website", - "PDS_Titan_Data_Archive_Website", - "PDS_Toolkits_Website", - "PDS_USGS_Pilot_Website", - "PDS_Uranus_Data_Archive_Website", - "PDS_Users_Guides_Website", - "PDS_Venus_Archive_Page_Website", - "PDS_Venus_Orbital_Data_Explorer_Website", - "PDS_Virtual_Astronaut_Website", - "PDS_Web_Chronos_Website", - "PDS_Wind_Tunnel_Particle_Threshold_Speed_Data_Website", - "PyHC_Website", - "SERVIR_Global", - "SPASE_JSON", - "SPASE_Website", - "SPEDAS_Website", - "Sea_Level_Change", - "Small_Bodies_Node", - "Socioeconomic_Data_and_Applications_Center_SEDAC", - "Solar_Data_Analysis_Center_SDAC", - "Solar_Physics_Group", - "Space_Biology_Science_Digest", - "Space_Physics_Data_Facility", - "Space_Place", - "TASKBOOK_Website", - "VEDA_Dashboard", - "VEDA_STAC_Catalog", - "algorithm_theoretical_basis_documents", - "archived_synthetic_data", - "astrogeology_analysis_ready_data", - "astroquery_api_search_mast_queries", - "co_plotter", - "contributed_datasets_in_the_exoplanet_archive", - "coordinate_calculator", - "corot_exoplanet_archive_etss_data_sets", - "earth_observer_publications", - "earth_observing_dashboard", - "earth_science_decadal_surveys", - "emac_exoplanet_modeling_and_analysis_center", - "eos_mission_page", - "exo_mast", - "exoplanet_atmosphere_observability_table", - "exoplanet_opacities_database", - "extinction_calculator", - "f_prime", - "fire_information_for_resource_management_system_firms", - "gcn_circulars", - "gcn_missions_instruments_and_facilities", - "general_coordinates_network_gcn", - "giss_datasets_and_derived_materials", - "giss_publication_list", - "giss_software_tools", - "goddard_institute_for_space_studies", - "heasarc_browse_batch_interface", - "heasarc_download_scripts", - "high_level_science_products", - "hubble_source_catalog_search", - "igwn_public_alerts_user_guide", - "interactive_multiinstrument_database_of_solar_flares", - "ipac_table_validator", - "koa_program_friendly_image_access_service", - "lbti_archive", - "mars_target_encyclopedia_mte", - "mast", - "mast_api_search", - "mast_hubble_search", - "mast_portal", - "mast_query_casjobs", - "mast_web_services", - "montage_mosaic_engine", - "my_nasa_data", - "naif", - "nasa_2023_climate_strategy", - "nasa_carbon_monitoring_system", - "nasa_global_climate_change", - "nasa_power", - "nasa_science_missions_earth", - "nasa_wavelength", - "neid_archive", - "neid_solar_radial_velocity_archive", - "nexsci", - "ntrs_api", - "our_changing_planet_the_view_from_space_images", - "pan_starrs_catalog", - "pan_starrs_catalog_api", - "pds_rings", - "planetary_image_galleries", - "pykoa", - "skiff_spectral_catalog_search", - "space_telescope_bibliographic_search", - "spectral_classes_of_like_stars", - "vao_datascope", - "velocity_calculator", - "venus_data_archive", - "virtual_observatory_information", - "z_mast_search", -] - -# after indexing for a few days, we discovered some possible optimizations and needed -# to restart the jobs -updated_list_20231003 = [ - "heasarc_download_scripts", - "ipac_table_validator", - "mast_api_search", - "montage_mosaic_engine", - "nasa_global_climate_change", - "neid_solar_radial_velocity_archive", - "pan_starrs_catalog_api", - "space_telescope_bibliographic_search", - "virtual_observatory_information", - "high_level_science_products", - "koa_program_friendly_image_access_service", - "mast_hubble_search", - "my_nasa_data", - "nasa_power", - "nexsci", - "pds_rings", - "spectral_classes_of_like_stars", - "z_mast_search", - "ASTRO_Planck_Cutout_Visualization_Website", - "ASTRO_exoMAST_API_Website", - "CASEI_Deployment", - "CEOS_API_M", - "ESSCOR_API", - "GENELAB_Github_DataProcessing", - "HAPI_API", - "Helioviewer_Website", - "NASA_Climate_Change", - "NASA_SPoRT", - "NasaEarthObservationWebsite", - "PDS_Annex_Data_Holdings_Website", - "PDS_Astropedia_Lunar_and_Planetary_Cartographic_Catalog_Website", - "PDS_Cassini_Mission_Enceladus_(Saturn_II)_Website", - "PDS_Cassini_Mission_Tethys_(Saturn_III)_Website", - "PDS_DIVINER_RDR_Query_Website", - "PDS_Data_Volumes_Index_Website", - "PDS_Errata_Website", - "PDS_High-Resolution_Transmission_Molecular_Absorption_Database_(HITRAN)_Website", - "PDS_Java_Mission-planning_and_Analysis_for_Remote_Sensing_(JMARS)_Website", - "PDS_LADEE_UVS_Calibrated_Data_Search", - "PDS_MAVEN_ACC_Data_Search", - "PDS_Mars_GCM_Website", - "PDS_Mercury_Orbital_Data_Explorer_Website", - "PDS_Missions_Website", - "PDS_NASA_Space_Science_Data_Coordinated_Archive_(NSSDCA)_Website", - "PDS_New_Horizons_Encounter_with_Pluto_Website", - "PDS_OPUS_Website", - "PDS_Outer_Planets_Icy_Satellites_Archive_Page_Website", - "PDS_PDS4_Training_Documents_Website", - "PDS_PDS_Software_Tools_Tutorial_and_Viewers_Website", - "PDS_Photojournal_Website", - "PDS_Recently_Archived_Volumes_Website", - "PDS_SPICE-enhanced_Cosmographia_Website", - "PDS_SPICE_Toolkit_Website", - "PDS_Small_Bodies_Image_Browser_Website", - "PDS_Titan_Data_Archive_Website", - "PDS_Venus_Archive_Page_Website", - "PyHC_Website", - "Sea_Level_Change", - "Space_Biology_Science_Digest", - "VEDA_STAC_Catalog", - "co_plotter", - "earth_observing_dashboard", - "exoplanet_atmosphere_observability_table", - "gcn_circulars", - "giss_software_tools", - "hubble_source_catalog_search", - "lbti_archive", - "mast_portal", - "naif", - "nasa_science_missions_earth", - "ntrs_api", - "planetary_image_galleries", - "vao_datascope", - "Space_Physics_Data_Facility", - "algorithm_theoretical_basis_documents", - "contributed_datasets_in_the_exoplanet_archive", - "earth_science_decadal_surveys", - "exoplanet_opacities_database", - "gcn_missions_instruments_and_facilities", - "goddard_institute_for_space_studies", - "igwn_public_alerts_user_guide", - "mars_target_encyclopedia_mte", - "mast_query_casjobs", - "nasa_2023_climate_strategy", - "nasa_wavelength", - "our_changing_planet_the_view_from_space_images", - "pykoa", - "velocity_calculator", - "ASTRO_NASA_Exoplanet_Archive_Documents_Website", - "ASTRO_TAP_Search_Website", - "Autoplot_Website", - "CASEI_Platform", - "CODE_NASA_API", - "GCIS_BOOKS_API", - "GENELAB_Github_Training", - "Helio_Events_Knowledgebase_Website", - "LSDA_Website_Trial", - "NASA_Earth_Observatory", - "NASA_Worldview", - "PDS_API_Legacy_All", - "PDS_Astrogeology_Website", - "PDS_CRISM_Analysis_Toolkit_(CAT)_Website", - "PDS_Cassini_Mission_Mimas_(Saturn_I)_Website", - "PDS_Collision_Induced_Absorption_Model_Website", - "PDS_Data_Dictionary_Search", - "PDS_Dawn_Mission_to_Ceres_Website", - "PDS_Geosciences_Data_Holdings_Website", - "PDS_ISIS_Website", - "PDS_Jupiter_Data_Archive_Website", - "PDS_LOLA_RDR_Query_Website", - "PDS_MRO_Coordinated_Observation_Website", - "PDS_Mars_Orbital_Data_Explorer_Website", - "PDS_Metadata_Injector_for_PDS_Labels_Website", - "PDS_NASA_Science_Earths_Moon_Website", - "PDS_NEAR_Shoemaker_Mission_to_433_Eros_Website", - "PDS_Notebook_Website", - "PDS_Object_Access_Library_Website", - "PDS_PDS4_Documents_Website", - "PDS_PDS_Atmospheres_Data_Set_Catalog_Website", - "PDS_PPI_Documents_Website", - "PDS_Planetary_Science_Tools_Website", - "PDS_Rings_Website", - "PDS_SPICE_Programming_Lessons_Website", - "PDS_SPICE_Utility_and_Application_Programs_Website", - "PDS_Subscription_Service_Website", - "PDS_USGS_Pilot_Website", - "PDS_Virtual_Astronaut_Website", - "SPASE_JSON", - "Socioeconomic_Data_and_Applications_Center_SEDAC", - "Space_Place", - "archived_synthetic_data", - "coordinate_calculator", - "emac_exoplanet_modeling_and_analysis_center", - "extinction_calculator", - "general_coordinates_network_gcn", - "heasarc_browse_batch_interface", - "interactive_multiinstrument_database_of_solar_flares", - "mast", - "mast_web_services", - "nasa_carbon_monitoring_system", - "neid_archive", - "pan_starrs_catalog", - "skiff_spectral_catalog_search", - "venus_data_archive", - # long jobs... - "giss_datasets_and_derived_materials", - "ASTRO_Missions_and_Data_Website", - "Small_Bodies_Node", - "ASTRO_Image_Cutouts_Website", -] - -update_base_href = [ - "SERVIR_Global", - "Sea_Level_Change", - "Solar_Physics_Group", - "Space_Physics_Data_Facility", - "astroquery_api_search_mast_queries", - "contributed_datasets_in_the_exoplanet_archive", - "corot_exoplanet_archive_etss_data_sets", - "exo_mast", - "exoplanet_atmosphere_observability_table", - "extinction_calculator", - "gcn_circulars", - "interactive_multiinstrument_database_of_solar_flares", - "mast", - "mast_api_search", - "mast_portal", - "neid_archive", - "our_changing_planet_the_view_from_space_images", - "pan_starrs_catalog", - "pan_starrs_catalog_api", - "space_telescope_bibliographic_search", - "spectral_classes_of_like_stars", - "vao_datascope", - "velocity_calculator", - "z_mast_search", -] - -deploy_to_prod_20231027 = [ - "interactive_multiinstrument_database_of_solar_flares", - "SPEDAS_Website", - "Helioviewer_Documentation_Website", - "GENELAB_Publications_Website", - "contributed_datasets_in_the_exoplanet_archive", - "corot_exoplanet_archive_etss_data_sets", - "exo_mast", - "exoplanet_atmosphere_observability_table", - "exoplanet_opacities_database", - "mast", -] - -sources_to_delete_20231031 = [ - "PDS_Odyssey_GRS_Data_Node_Website", - "PDS_Map-a-Planet_(MAP)_Website", - "PDS_Projection_on_the_Web_(POW)_Service_Website", - "PDS_LADEE_NMS_Derived_Data_Search", - "PDS_LADEE_NMS_Calibrated_Data_Search", - "PDS_Wind_Tunnel_Particle_Threshold_Speed_Data_Website", - "exofop_k2_campaign_9", - "GENELAB_METADATA_Website", - "pds_geosciences_node_community", - "2mass_batch_image_service", - "2mass_api_search", - "2mass_interactive_image_service", - "2mass_image_inventory_search", - "image_and_spectrum_server_atlas", - "atlas_api_search", - "background_model", - "galactic_dust_reddening_and_extinction", - "ASTRO_Finder_Chart_Website", - "irsa_api", - "herschel_data_search", - "hires", - "wise_neowise_coadder", - "image_validation", - "object_coordinate_lookup", - "most", - "scanpi", - "swas_spectrum_server", - "swas_api_search", - "planck_data_tools", - "irsa_votable_access_protocol", - "vo_simple_cone_search", - "ssa_queries", - "irsa_idl_tools", - "PDS_Web_Chronos_Website", - "PDS_Subset_Tool_Website", - "PDS_SPICE-enhanced_Cosmographia_Website", - "PDS_SPICE_Archives_Website", - "PDS_Operational_Flight_Other_Project_Kernels_Website", - "PDS_SPICE_Toolkit_Documentation_Website", - "PDS_SPICE_Programming_Lessons_Website", - "PDS_SPICE_Toolkit_Website", - "PDS_SPICE_Tutorials_Website", - "PDS_SPICE_Utility_and_Application_Programs_Website", - "PDS_Generic_Kernels_Website", - "PDS_Niels_Bohr_Institute_Website", - "PDS_PDS_Atmospheres_Data_Set_Catalog_Website", - "PDS_LADEE_UVS_Calibrated_Data_Search", - "PDS_Recently_Archived_Volumes_Website", - "PDS_Object_Access_Library_Website", - "ftp_access", - "index", - "PDS_Data_Archive_Website", - "PDS_All_Data_Holdings_Website", - "PDS_PDS_Software_Tools_Tutorial_and_Viewers_Website", - "PDS_Data_Volumes_Index_Website", - "PDS_PPI_Documents_Website", - "PDS_Missions_Website", - "PDS_Errata_Website", - "PDS_Data_Volumes_Website", - "PDS_Models_and_Simulations_Website", - "PDS_Rings_Website", - "viewmaster_data_archives_calibrated", - "viewmaster_data_archives_diagrams", - "viewmaster_data_archives_previews", - "viewmaster_data_archives_volumes", - "viewmaster_data_calibrated", - "viewmaster_data_diagrams", - "viewmaster_documents", - "viewmaster_data_previews", - "viewmaster_data_volumes", - "small_bodies_node_data_by_type_of_observation", - "PDS_Mission_Data_Archive_Website", - "small_bodies_node_data_by_target_type", - "data_set_status", - "PDS_Subscription_Service_Website", - "PDS_USGS_Pilot_Website", - "PDS_OSIRIS-REx_Mission_to_Bennu_Website", - "PDS_Dawn_Mission_to_Ceres_Website", - "PDS_Cassini_Mission_Dione_(Saturn_IV)_Website", - "PDS_Cassini_Mission_Enceladus_(Saturn_II)_Website", - "PDS_NEAR_Shoemaker_Mission_to_433_Eros_Website", - "PDS_Cassini_Mission_Iapetus_(Saturn_VIII)_Website", - "PDS_Cassini_Mission_Mimas_(Saturn_I)_Website", - "PDS_Phoebe_Saturn_IX_Website", - "PDS_New_Horizons_Encounter_with_Pluto_Website", - "PDS_Cassini_Mission_Rhea_(Saturn_V)_Website", - "PDS_Cassini_Mission_Tethys_(Saturn_III)_Website", - "PDS_Dawn_Mission_to_Vesta_Website", - "PDS_SBIB_3D_Website", - "PDS_Cassini_Mission_Saturn_Small_Satellites_Website", - "PDS_Small_Bodies_Data_Ferret_Website", - "PDS_Astromat_Astromaterials_Data_System_Website", - "PDS_ISIS_Website", - "navo_registry", - "PDS_Planetary_Data_System_(PDS)_Website", - "ASTRO_Multi_Website", - "LSDA_Website_Trial", - "LSDA_Website_Trial2", - "PDS_PDS_Documentation_Website", - "PDS_Data_Portal_Website", - "PDS_Image_Atlas_Website", - "PDS_Imaging_Software_Website", - "PDS_Users_Guides_Website", - "GENELAB_Publications_Website", -] - - -sources_to_index_20240618 = [ - "Mission_Independent_Data_Layer", - "koa_keck_observatory_archive", - "interstellar_mapping_and_acceleration_probe", - "explorers_and_heliophysics_projects_division", - "interstellar_boundary_explorer_ibex", - "polarimeter_to_unify_the_corona_and_heliosphere", - "helioswarm", - "voyager", - "earth_science_office_at_msfc", - "earth_system_science_pathfinder", - "cloudsat", - "carve_carbon_in_arctic_reservoirs_vulnerability_experiment", - "a_train_the_afternoon_constellation", - "cygnss_cyclone_global_navigation_satellite_system", - "dcotss_dynamics_and_chemistry_of_the_summer_stratosphere", - "delta_x", - "center_for_climate_sciences", -] - -sources_to_index_20240717 = [ - "nasa_ames_intelligent_systems_division_data", - "chandra_x_ray_observatory", - "hi_c_at_msfc", - "tandem_reconnection_and_cusp_electrodynamics_reconnaissance_satellites", - "sungrazer_project", - "solar_system_dynamics", - "lowell_minor_planet_services", - "explorer_1", - "earth_sciences_at_gsfc", - "earth_science_at_jpl", - "ecostress", - "seabass", - "ecostress_spectral_library", - "unite_unistellar_network_investigating_tess_exoplanets", - "satellite_situation_center_sscweb_system_and_services", - "the_astrophysics_astrochemistry_lab", - "hubble_space_telescope", - "galprop", - "chandra_x_ray_center_cxc", -] - -sources_to_index_20240730 = [ - "gcn_circulars", - "gamma_ray_astrophysics_at_the_nsstc", - "asteroid_lightcurve_photometry_database", - "orbital_data_explorer", - "asdc_misr", - "archived_gcn", - "ghrc_global_hydrometeorology_resource_center", - "NASA_Earth_Observatory", - "aurorasaurus_reporting_auroras_from_the_ground_up", - "hinode_at_msfc", - "osiris_rex_asteroid_sample_return_mission", - "grace_and_grace_fo_groundwater_and_soil_moisture_conditions", - "sun_climate_powered_by_solar_irradiance", - "icesat_2_ice_cloud_and_land_elevation_satellite_2", - "spruce_spruce_and_peatland_responses_under_changing_environments", - "tess_transitioning_exoplanet_survey_satellite", - "iasc_international_astronomical_search_collaboration", - "debit_dynamic_eclipse_broadcast_initiative", - "astropix", - "Space_Physics_Data_Facility", - "PDS_Photojournal_Website", -] - -sources_with_neural_and_plugin = [ - "Geosciences_Node", - "Helioviewer_Documentation_Website", - "International_Heliophysics_Data_Environment_Alliance_Overview_IHDEA", - "act_america_atmospheric_carbon_and_transport_america", - "activate_aerosol_cloud_meteorology_interactions_over_the_western_atlantic_experiment", - "advanced_composition_explorer", - "airmoss_airborne_microwave_observatory_of_subcanopy_and_subsurface_at_jpl", - "aria_advanced_rapid_imaging_and_analysis", - "atmospheric_imaging_assembly", - "explorer_program_acquisition", - "fast_plasma_investigation", - "fermi_at_gsfc", - "hawc_observatory", - "hinode_solar_b", - "incus_investigation_of_convective_updrafts", - "interactive_nasa_space_physics_ionosphere_radio_experiments_inspire", - "iris_interface_region_imaging_spectrograph", - "jwst_user_documentation", - "ldas_land_data_assimilatin_system", - "lisa_consortium", - "mdscc_deep_space_network", - "microobservatory_robotic_telescope_network", - "naif", - "nasa_applied_sciences", - "nasa_arcgis_online", - "nasa_infrared_telescope_facility_irtf", - "nasa_sounding_rocket_program", - "parker_solar_probe", - "pds_cartography_and_imaging_sciences_node", - "pds_website", - "physics_of_the_cosmos", - "ppi_node", - "pps_precipitation_processing_system", - "radio_jove", - "rhessi_homepage", - "sampex_data_center", - "sdo_solar_dynamics_observatory", - "solar_terrestrial_probes_program", - "starchild_a_learning_center_for_young_astronomers", - "stereo_at_gsfc", - "stereo_cor1", - "the_new_great_observatories", - "treasure_map", - "vector_electric_field_instrument", - "virbo", - "virtual_wave_observatory", -] - -sources_to_index_test_grid = [ - "a_train_the_afternoon_constellation", - "act_america_atmospheric_carbon_and_transport_america", - "activate_aerosol_cloud_meteorology_interactions_over_the_western_atlantic_experiment", - "advanced_composition_explorer", - "Advancing_the_science_of_heliophysics", - "airmoss_airborne_microwave_observatory_of_subcanopy_and_subsurface_at_jpl", - "archived_gcn", - "aria_advanced_rapid_imaging_and_analysis", - "asdc_misr", - "asteroid_lightcurve_photometry_database", - "astronomy_picture_of_the_day", - "astrophysics_source_code_library", - "astropix", - "astropy", - "atmospheric_imaging_assembly", - "aurorasaurus_reporting_auroras_from_the_ground_up", - "carve_carbon_in_arctic_reservoirs_vulnerability_experiment", - "cdaw_data_center", - "center_for_climate_sciences", - "chandra_x_ray_center_cxc", - "chandra_x_ray_observatory", - "cii_hosted_payload_opportunity_online_database", - "cloudsat", - "cosmic_data_stories", - "cygnss_cyclone_global_navigation_satellite_system", - "dcotss_dynamics_and_chemistry_of_the_summer_stratosphere", - "debit_dynamic_eclipse_broadcast_initiative", - "delta_x", - "dscovr_epic_earth_polychromatic_imaging_camera", - "earth_science_at_jpl", - "earth_science_office_at_msfc", - "earth_system_science_pathfinder", - "ecostress", - "ecostress_spectral_library", - "explorer_1", - "explorer_program_acquisition", - "explorers_and_heliophysics_projects_division", - "fast_plasma_investigation", - "fermi_at_gsfc", - "galprop", - "gamma_ray_astrophysics_at_the_nsstc", - "gamma_ray_data_tools_core_package", - "gamma_ray_data_tools_github", - "Geosciences_Node", - "ghrc_global_hydrometeorology_resource_center", - "global_sulfur_dioxide_monitoring", - "gmao_fluid", - "grace_and_grace_fo_groundwater_and_soil_moisture_conditions", - "hawc_observatory", - "HelioAnalytics", - "helionauts", - "heliophysics_digital_observatory", - "heliophysics_system_observatory_connect", - "helioswarm", - "Helioviewer_Documentation_Website", - "hi_c_at_msfc", - "hinode_at_msfc", - "hinode_solar_b", - "hpde_github", - "hubble_space_telescope", - "icesat_2_ice_cloud_and_land_elevation_satellite_2", - "incus_investigation_of_convective_updrafts", - "interactive_nasa_space_physics_ionosphere_radio_experiments_inspire", - "International_Heliophysics_Data_Environment_Alliance_Overview_IHDEA", - "interstellar_boundary_explorer_ibex", - "interstellar_mapping_and_acceleration_probe", - "iris_interface_region_imaging_spectrograph", - "JWST_User_Documentation", - "koa_keck_observatory_archive", - "land_processes_distributed_active_archive_center", - "ldas_land_data_assimilatin_system", - "lisa_consortium", - "Living_with_a_Star", - "lowell_minor_planet_services", - "magnetospheric_multiscale_satellites", - "mdscc_deep_space_network", - "microobservatory_robotic_telescope_network", - "Mission_Independent_Data_Layer", - "naif", - "nasa_ames_intelligent_systems_division_data", - "nasa_applied_sciences", - "nasa_arcgis_online", - "NASA_Earth_Observatory", - "nasa_infrared_telescope_facility_irtf", - "nasa_sounding_rocket_program", - "nasa_visible_earth", - "national_space_weather_program", - "neil_gehrel_s_swift_observatory", - "orbital_data_explorer", - "osiris_rex_asteroid_sample_return_mission", - "parker_solar_probe", - "pds_atmospheres", - "pds_cartography_and_imaging_sciences_node", - "pds_nasa_solar_system_treks_website", - "PDS_Photojournal_Website", - "pds_rings", - "pds_website", - "physics_of_the_cosmos", - "polarimeter_to_unify_the_corona_and_heliosphere", - "ppi_node", - "pps_precipitation_processing_system", - "radio_jove", - "rhessi_homepage", - "sampex_data_center", - "satellite_situation_center_sscweb_system_and_services", - "sdo_solar_dynamics_observatory", - "seabass", - "sep_instrument_suite", - "Small_Bodies_Node", - "solar_and_heliospheric_observatory_soho", - "Solar_Physics_Group", - "solar_system_dynamics", - "solar_terrestrial_probes_program", - "spruce_spruce_and_peatland_responses_under_changing_environments", - "starchild_a_learning_center_for_young_astronomers", - "stereo_at_gsfc", - "stereo_cor1", - "sun_climate_powered_by_solar_irradiance", - "sungrazer_project", - "tandem_reconnection_and_cusp_electrodynamics_reconnaissance_satellites", - "TASKBOOK_Website", - "tess_transitioning_exoplanet_survey_satellite", - "the_astrophysics_astrochemistry_lab", - "the_new_great_observatories", - "treasure_map", - "unite_unistellar_network_investigating_tess_exoplanets", - "Van_Allen_Probes", - "vector_electric_field_instrument", - "virtual_solar_observatory", - "virtual_wave_observatory", - "voyager", - "Voyager_Cosmic_Ray_Subsystem", - "WIND_Spacecraft", -] - -missing_indexers = [ - "Advancing_the_science_of_heliophysics", - "HelioAnalytics", - "JWST_User_Documentation", - "Living_with_a_Star", - "pds_nasa_solar_system_treks_website", - "solar_and_heliospheric_observatory_soho", -] - -sources_to_index_test_grid_20240809 = [s for s in sources_to_index_test_grid if s not in missing_indexers] diff --git a/config_generation/tests/test_config_generation_pipeline.py b/config_generation/tests/test_config_generation_pipeline.py deleted file mode 100644 index ebde072f..00000000 --- a/config_generation/tests/test_config_generation_pipeline.py +++ /dev/null @@ -1,90 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from django.test import TestCase - -from sde_collections.models.collection import Collection -from sde_collections.models.collection_choice_fields import WorkflowStatusChoices - -""" -Workflow status change → Opens template → Applies XML transformation → Writes to GitHub. - -- When the `workflow_status` changes, it triggers the relevant config creation method. -- The method reads an template and processes it using `XmlEditor`. -- `XmlEditor` modifies the template by injecting collection-specific values and transformations. -- The generated XML is passed to `_write_to_github()`, which commits it directly to GitHub. - -Note: This test verifies that the correct methods are triggered and XML content is passed to GitHub. -The actual XML structure and correctness are tested separately in `test_db_xml.py`. -""" - - -class TestConfigCreation(TestCase): - def setUp(self): - self.collection = Collection.objects.create( - name="Test Collection", division="1", workflow_status=WorkflowStatusChoices.RESEARCH_IN_PROGRESS - ) - - @patch("sde_collections.utils.github_helper.GitHubHandler") # Mock GitHubHandler - @patch("sde_collections.models.collection.Collection._write_to_github") - @patch("sde_collections.models.collection.XmlEditor") - def test_ready_for_engineering_triggers_config_and_job_creation( - self, MockXmlEditor, mock_write_to_github, MockGitHubHandler - ): - """ - When the collection's workflow status is updated to READY_FOR_ENGINEERING, - it should trigger the creation of scraper configuration and job files. - """ - # Mock GitHubHandler to avoid actual API calls - mock_github_instance = MockGitHubHandler.return_value - mock_github_instance.create_file.return_value = None - mock_github_instance.create_or_update_file.return_value = None - - # Set up the XmlEditor mock for both config and job - mock_editor_instance = MockXmlEditor.return_value - mock_editor_instance.convert_template_to_scraper.return_value = "config_data" - mock_editor_instance.convert_template_to_job.return_value = "job_data" - - # Simulate the status change to READY_FOR_ENGINEERING - self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING - self.collection.save() - - # Verify that the XML for both config and job are generated and written to GitHub - expected_calls = [ - call(self.collection._scraper_config_path, "config_data", False), - call(self.collection._scraper_job_path, "job_data", False), - ] - mock_write_to_github.assert_has_calls(expected_calls, any_order=True) - - @patch("sde_collections.models.collection.GitHubHandler") # Mock GitHubHandler in the correct module path - @patch("sde_collections.models.collection.Collection._write_to_github") - @patch("sde_collections.models.collection.XmlEditor") - def test_ready_for_curation_triggers_indexer_config_and_job_creation( - self, MockXmlEditor, mock_write_to_github, MockGitHubHandler - ): - """ - When the collection's workflow status is updated to READY_FOR_CURATION, - it should trigger indexer config and job creation methods. - """ - # Mock GitHubHandler to avoid actual API calls - mock_github_instance = MockGitHubHandler.return_value - mock_github_instance.check_file_exists.return_value = True # Assume scraper exists - mock_github_instance._get_file_contents.return_value = MagicMock() - mock_github_instance._get_file_contents.return_value.decoded_content = ( - b"Mock Data" - ) - - # Set up the XmlEditor mock for both config and job - mock_editor_instance = MockXmlEditor.return_value - mock_editor_instance.convert_template_to_indexer.return_value = "config_data" - mock_editor_instance.convert_template_to_job.return_value = "job_data" - - # Simulate the status change to READY_FOR_CURATION - self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION - self.collection.save() - - # Verify that the XML for both indexer config and job are generated and written to GitHub - expected_calls = [ - call(self.collection._indexer_config_path, "config_data", True), - call(self.collection._indexer_job_path, "job_data", False), - ] - mock_write_to_github.assert_has_calls(expected_calls, any_order=True) diff --git a/config_generation/tests/test_db_to_xml.py b/config_generation/tests/test_db_to_xml.py deleted file mode 100644 index 197c6044..00000000 --- a/config_generation/tests/test_db_to_xml.py +++ /dev/null @@ -1,142 +0,0 @@ -# docker-compose -f local.yml run --rm django pytest config_generation/tests/test_db_to_xml.py -from xml.etree.ElementTree import ElementTree, ParseError, fromstring - -import pytest - -from ..db_to_xml import XmlEditor - - -def xmls_equal(xml1, xml2): - """ - Check the structural and textual equality of two XML strings. - - Parameters: - - xml1, xml2 (str): The XML strings to compare. - - Returns: - - bool: True if XMLs are structurally and textually equal, False otherwise. - """ - - def elements_equal(e1, e2): - # Check tag and text - if e1.tag != e2.tag or (e1.text or "").strip() != (e2.text or "").strip(): - return False - - # Check attributes (ignoring order) - if sorted(e1.attrib.items()) != sorted(e2.attrib.items()): - return False - - # Check children - if len(e1) != len(e2): - return False - return all(elements_equal(c1, c2) for c1, c2 in zip(e1, e2)) - - tree1 = ElementTree(fromstring(xml1)) - tree2 = ElementTree(fromstring(xml2)) - - return elements_equal(tree1.getroot(), tree2.getroot()) - - -# Tests for valid and invalid XML initializations -def test_valid_xml_initialization(): - xml_string = "Test" - editor = XmlEditor(xml_string) - assert editor.get_tag_value("child") == ["Test"] - - -def test_invalid_xml_initialization(): - with pytest.raises(ParseError): - XmlEditor("") - - -# Test retrieval of single and multiple tag values -def test_get_single_tag_value(): - xml_string = "Test" - editor = XmlEditor(xml_string) - assert editor.get_tag_value("child", strict=True) == "Test" - - -def test_get_nonexistent_tag_value(): - xml_string = "Test" - editor = XmlEditor(xml_string) - assert editor.get_tag_value("nonexistent", strict=False) == [] - - -def test_get_tag_value_strict_multiple_elements(): - xml_string = "OneTwo" - editor = XmlEditor(xml_string) - with pytest.raises(ValueError): - editor.get_tag_value("child", strict=True) - - -# Test updating and adding XML elements -def test_update_existing_element(): - xml_string = "Old" - editor = XmlEditor(xml_string) - editor.update_or_add_element_value("child", "New") - updated_xml = editor.update_config_xml() - assert "New" in updated_xml and "Old" not in updated_xml - - -def test_add_new_element(): - xml_string = "" - editor = XmlEditor(xml_string) - editor.update_or_add_element_value("newchild", "Value") - updated_xml = editor.update_config_xml() - assert "Value" in updated_xml and "Value" in updated_xml - - -def test_add_third_level_hierarchy(): - xml_string = "" - editor = XmlEditor(xml_string) - editor.update_or_add_element_value("parent/child/grandchild", "DeeplyNested") - updated_xml = editor.update_config_xml() - root = fromstring(updated_xml) - grandchild = root.find(".//grandchild") - assert grandchild is not None, "Grandchild element not found" - assert grandchild.text == "DeeplyNested", "Grandchild does not contain the correct text" - - # Check complete path - parent = root.find(".//parent/child/grandchild") - assert parent is not None, "Complete path to grandchild not found" - assert parent.text == "DeeplyNested", "Complete path to grandchild does not contain correct text" - - -# Test transformations and generic mapping -def test_convert_indexer_to_scraper_transformation(): - xml_string = """Indexer""" - editor = XmlEditor(xml_string) - editor.convert_indexer_to_scraper() - updated_xml = editor.update_config_xml() - assert "SMD_Plugins/Sinequa.Plugin.ListCandidateUrls" in updated_xml - assert "Indexer" not in updated_xml - - -def test_generic_mapping_addition(): - xml_string = "" - editor = XmlEditor(xml_string) - editor._generic_mapping(name="id", value="doc.url1", selection="url1") - updated_xml = editor.update_config_xml() - assert "" in updated_xml - assert "id" in updated_xml - assert "doc.url1" in updated_xml - - -# Test XML serialization with headers -def test_xml_serialization_with_header(): - xml_string = "Value" - editor = XmlEditor(xml_string) - xml_output = editor.update_config_xml() - assert '' in xml_output - assert "" in xml_output and "Value" in xml_output - - -# Test handling multiple changes accumulation -def test_multiple_changes_accumulation(): - xml_string = "Initial" - editor = XmlEditor(xml_string) - editor.update_or_add_element_value("child", "Modified") - editor.update_or_add_element_value("newchild", "Added") - updated_xml = editor.update_config_xml() - assert "Modified" in updated_xml and "Added" in updated_xml - assert "Initial" not in updated_xml diff --git a/config_generation/xmls/command_template.xml b/config_generation/xmls/command_template.xml deleted file mode 100644 index 17a63413..00000000 --- a/config_generation/xmls/command_template.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - 2 - - updatecolumns - - - Auto - - - true - - false - - - true - - - true - git - - false - - @@ScienceMissionDirectorate - - false - - - diff --git a/config_generation/xmls/delete_template.xml b/config_generation/xmls/delete_template.xml deleted file mode 100644 index bbbde682..00000000 --- a/config_generation/xmls/delete_template.xml +++ /dev/null @@ -1,22 +0,0 @@ - - 2 - - sql - - - - - true - - false - - - - true - - - true - - - false - diff --git a/config_generation/xmls/example_exclude_and_title_patterns.xml b/config_generation/xmls/example_exclude_and_title_patterns.xml deleted file mode 100644 index de795376..00000000 --- a/config_generation/xmls/example_exclude_and_title_patterns.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - //*[@id="ssdBgWrapper"]/header - false - - - - - //*[@id="jwstFooter"] - false - - - - - - - title - "Images of James Web for Press" - - url1 = 'https://webb.nasa.gov/content/forPress/index.html' - - - - - - title - IfEmpty(Concat(xpath:/html/body/div/div/div/p[1]/font, ' ', - xpath:/html/body/div/div/div/p[1]/font[2]),xpath:/html/head/title) - Helio Events Title - - - - - -https://webb.nasa.gov/content/news/webbBuildStatusArchive.html - - - -https://webb.nasa.gov/content/forEducators/realworld* -*.rtf diff --git a/config_generation/xmls/indexer_template.xml b/config_generation/xmls/indexer_template.xml deleted file mode 100644 index 3559cdcc..00000000 --- a/config_generation/xmls/indexer_template.xml +++ /dev/null @@ -1,289 +0,0 @@ - - - crawler2 - - - - - - - 1 - - false - - SMD_Plugins/Sinequa.Plugin.WebCrawler_Index_URLList - 3 - - - - - - - - - - true - - true - - - - - - - true - false - false - true - true - false - true - true - false - true - true - true - true - false - - - - true - no - false - - false - false - false - false - false - - - false - true - true - true - false - false - true - false - false - false - false - false - false - false - - - - true - true - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - expBackoff+headers - false - - - - - - - - - - - - false - true - - - - - false - - - false - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - - true - - true - - - true - false - - - - - - INFO - - false - - true - false - - - - false - false - false - false - true - false - - - - false - false - - - false - false - false - - - - - - - - - false - false - false - - - - true - - - - - - false - false - false - - true - - false - true - true - false - false - false - false - - - - false - false - true - false - - - true - false - - - - true - - - - - - - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - false - - - - - - - - - - false - - - - - - false - - - - - - false - - - id - doc.url1 - - false - false - diff --git a/config_generation/xmls/indexing_template.xml b/config_generation/xmls/indexing_template.xml deleted file mode 100644 index 5faa8506..00000000 --- a/config_generation/xmls/indexing_template.xml +++ /dev/null @@ -1,285 +0,0 @@ - - - crawler2 - - - - - /your/treeroot/here/ - false - - - rtf;xml;jy;ico;gz;act;xsd - - - - - - - false - false - false - - - true - - - - - - false - false - false - - - - 20 - - false - - true - false - - - false - false - false - false - false - false - - - - false - false - - false - false - false - - - - - - - - - - - - - - - - false - true - true - false - false - false - false - - - - false - false - true - false - - - false - false - - - - true - - - - - - - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - - - - - - - - - false - - - - - - false - - - - - - false - - 3 - - your_url_here - - - true - true - true - 100 - 100000 - 100000 - 10 - -1 - -1 - true - false - false - false - false - false - true - true - false - true - true - true - true - false - - - true - no - false - - false - false - True - false - false - - - false - true - true - true - false - true - true - false - false - false - false - false - false - false - - - - true - true - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - false - - - - - - - - - - - false - true - - - - - false - - - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - true - - true - - - true - false - - - - - - id - doc.url1 - - - - - diff --git a/config_generation/xmls/job_command_template.xml b/config_generation/xmls/job_command_template.xml deleted file mode 100644 index d8203e0e..00000000 --- a/config_generation/xmls/job_command_template.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - command - - - false - - - - - - - - - - - - - false - - - - - - - false - - true - false - - - - collectioncache.GCMD.CASEI_Campaign - diff --git a/config_generation/xmls/job_template.xml b/config_generation/xmls/job_template.xml deleted file mode 100644 index 7763ecf1..00000000 --- a/config_generation/xmls/job_template.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - collection - - - - - false - - - - - - - - - - - - - false - - - - - - - false - - false - false - - - - diff --git a/config_generation/xmls/joblist_template.xml b/config_generation/xmls/joblist_template.xml deleted file mode 100644 index 72d79c13..00000000 --- a/config_generation/xmls/joblist_template.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - joblist - - - false - - - - - - - - - - - - - false - - - - - 20 - - false - - false - false - - - - true - false - - diff --git a/config_generation/xmls/json_url_indexing_template.xml b/config_generation/xmls/json_url_indexing_template.xml deleted file mode 100644 index db1b9579..00000000 --- a/config_generation/xmls/json_url_indexing_template.xml +++ /dev/null @@ -1,373 +0,0 @@ - - - - json.v2 - - - - - /Testing/Solar System Exploration/ - false - - htm;html - - - *index.html;*.xml - - - - - false - false - false - - - true - - - - - - false - false - false - 0 - - - 20 - - false - - false - false - - - false - false - false - false - true - false - - - - false - false - true - - - false - false - false - - - - - - - - - - - - - - - - false - true - true - false - false - false - - - - false - false - true - false - - - true - false - - - - true - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - - - - - - - - - false - - - - - - false - - Balanced - - - - false - - - - $.url_list.url - - false - - false - - solar - $.data - $.solar - - - - - - true - url_list - GET - https://sde-indexing-helper.nasa-impact.net/candidate-urls-api/CONFIG_FOLDER_HERE/ - $.results - - - https://sde-indexing-helper.nasa-impact.net/candidate-urls-api/CONFIG_FOLDER_HERE/?format=json&page={Add(datasource.url_list.pagenumber,1)} - datasource.url_list.pagenumber=132 - - - true - BothModes - - - - - - - false - - AddOrUpdate - - - - - - - - - - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - false - - - - - - - - - false - true - - - - - false - - - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - - true - - true - - 80 - true - false - - - - - - - 6 - - - - true - true - true - 100 - 100000 - 100000 - 10 - -1 - -1 - true - false - false - false - false - false - true - true - false - true - true - true - true - false - 1 - 0 ms - true - no - false - - false - false - false - false - false - - - false - true - true - true - false - false - true - false - false - false - false - false - false - false - - - - true - true - 1 - - false - - - false - - - - - - - {"class":"go.GraphLinksModel","nodeKeyProperty":":id","nodeCategoryProperty":":type","linkCategoryProperty":":type","linkFromPortIdProperty":":fromPort","linkToPortIdProperty":":toPort","linkFromKeyProperty":":from","linkToKeyProperty":":to","nodeDataArray":[{":id":"start",":type":"Start","Name":"Start"},{":id":"1url_list_sendrequest",":type":"DataSourceSendRequest","Name":"'url_list'\nSend request 'GET'\nhttps:\/\n\/sde-indexing-help\ner.nasa-impact.net\n\/candidate-urls-ap\ni\/nasa_power\/"},{":id":"1url_list_foreach",":type":"DataSourceForEach","Name":"For each item\n$.results\nAdd (or update)"},{":id":"to_indexer_1url_list_foreach",":type":"End","Name":"To indexer"}],"linkDataArray":[{":from":"start",":to":"1url_list_sendrequest",":fromPort":"out",":toPort":"in",":type":"ds_to_ds_link"},{":from":"1url_list_sendrequest",":to":"1url_list_foreach",":fromPort":"out",":toPort":"in",":type":"ds_to_ds_link"},{":from":"1url_list_foreach",":to":"to_indexer_1url_list_foreach",":fromPort":"out",":toPort":"in",":type":"ds_to_ds_link","Name":"To indexer"}]} - false - - - - - id - $.url_list.url - id - - - - - version - $.url_list.hash - version - - - - - title - $.url_list.title - Data item title - - - - - sourcestr56 - $.url_list.document_type - Data item document type - - - - - fileext - $.url_list.file_extension - Data item file extension - - - - diff --git a/config_generation/xmls/scraper_template.xml b/config_generation/xmls/scraper_template.xml deleted file mode 100644 index baf596ea..00000000 --- a/config_generation/xmls/scraper_template.xml +++ /dev/null @@ -1,295 +0,0 @@ - - - crawler2 - - - - - - - 1 - - false - - 3 - - - - - html;htm;xlsx;xls;xlsm;doc;docx;ppt;pdf - - - - - - true - - true - true - true - - - - - - - true - false - false - true - true - false - true - true - false - true - true - true - true - false - - 0 ms - - true - no - false - - false - false - false - false - false - - - false - true - true - true - false - false - true - false - false - false - false - false - false - false - - - - true - true - - id - doc.url1 - - - title - doc.filename - - doc.fileext = "pdf" - - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - expBackoff+headers - false - - - - - - - - - - - - false - true - - - - - false - - - false - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - - true - - true - - - true - false - - - - - - INFO - - false - - true - false - - - - false - false - false - false - true - false - - - - false - false - - - false - false - false - - - - - - - - - false - false - false - - - - true - - - - - - false - false - false - - true - - false - true - true - false - false - false - false - - - - false - false - true - false - - - true - false - - - - true - - - - - - - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - false - - - - - - - - - - false - - - - - - false - - - - - - false - - diff --git a/default_scraper.xml b/default_scraper.xml deleted file mode 100644 index ef583b3b..00000000 --- a/default_scraper.xml +++ /dev/null @@ -1,287 +0,0 @@ - - - Default crawler to create a URL candidate list - - crawler2 - - - - - fake treeroot - false - SMD_Plugins/Sinequa.Plugin.ListCandidateUrls - - - - - - - - - false - false - false - - - true - - - - - - false - false - false - - - false - - true - false - - - false - false - false - false - true - false - - - - false - false - - false - false - false - - - - - - - - - - - - - - - - false - true - true - false - false - false - - - - false - false - true - false - - _Advanced - true - false - - - - true - - - - - - - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - - - - - - - - - false - - - - - - false - - - - - - false - - 3 - - enter your url here - - - true - true - true - 100 - 100000 - 100000 - 10 - -1 - -1 - true - false - false - false - false - false - true - true - false - true - true - true - true - false - 1 - 0 ms - true - no - false - - false - false - false - false - false - - - false - true - true - true - false - true - true - false - false - false - false - false - false - false - *.rtf,*.jy,*.xml,*.ico,*.gz,*.act - - - - true - true - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - true - - - - - - - - - - - false - true - - - - - false - - - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - true - - true - - 80 - true - false - - - - - false - false - - - id - doc.url1 - - - - - diff --git a/docs/architecture-decisions/testing_strategy.md b/docs/architecture-decisions/testing_strategy.md index a8754750..465f8998 100644 --- a/docs/architecture-decisions/testing_strategy.md +++ b/docs/architecture-decisions/testing_strategy.md @@ -1,184 +1,153 @@ ## Overview -As of early 2025, we have only recently been writing tests for new features, and have about 250 tests in total, mostly centered around the EJ portal, the reindexing process, and pattern applications. - -Although this covers much of the core system logic, there still remain a number of untested logical areas such as the config file generation, core project settings, frontend features, etc. - -This document outlines a testing strategy for the project, which will guide us towards adding tests in the most critical areas first, followed by a plan to fully cover the remaining areas. - -## Current Coverage -Using the coverage library, the following report was generated: -Name | Stmts | Miss | Cover | Missing -----------|--------------------------------------------------------------------------------------------------|-------|--------|-------- -config/__init__.py | 2 | 0 | 100% | -config/celery_app.py | 6 | 0 | 100% | -config/settings/__init__.py | 0 | 0 | 100% | -config/settings/base.py | 94 | 0 | 100% | -config/settings/local.py | 20 | 20 | 0% | 1-65 -config/settings/production.py | 48 | 48 | 0% | 1-162 -config/urls.py | 14 | 4 | 71% | 26-47 -config/wsgi.py | 8 | 8 | 0% | 17-36 -config_generation/__init__.py | 0 | 0 | 100% | -config_generation/api.py | 34 | 34 | 0% | 1-88 -config_generation/config_example.py | 15 | 15 | 0% | 1-69 -config_generation/db_to_xml.py | 203 | 133 | 34% | 45, 47, 50, 96, 119-125, 129-136, 142-149, 197-200, 206-214, 225-230, 242-271, 274-278, 285-292, 303-308, 311, 317, 326-332, 342-349, 361-368, 371-374, 377-378, 382-390, 393-399, 402-412, 415-429 -config_generation/db_to_xml_file_based.py | 52 | 52 | 0% | 4-119 -config_generation/delete_config_folders.py | 24 | 24 | 0% | 9-50 -config_generation/delete_server_content.py | 12 | 12 | 0% | 3-25 -config_generation/delete_webapp_collections.py | 5 | 5 | 0% | 6-12 -config_generation/export_collections.py | 36 | 36 | 0% | 1-73 -config_generation/export_whole_index.py | 28 | 28 | 0% | 1-58 -config_generation/generate_collection_list.py | 29 | 29 | 0% | 8-69 -config_generation/generate_commands.py | 41 | 41 | 0% | 6-87 -config_generation/generate_emac_indexer.py | 24 | 24 | 0% | 1-81 -config_generation/generate_jobs.py | 42 | 42 | 0% | 8-100 -config_generation/generate_scrapers.py | 15 | 15 | 0% | 2-54 -config_generation/minimum_api.py | 33 | 33 | 0% | 1-81 -config_generation/preprocess_sources.py | 25 | 25 | 0% | 1-50 -config_generation/sources_to_scrape.py | 28 | 28 | 0% | 2-1631 -docs/__init__.py | 0 | 0 | 100% | -docs/conf.py | 17 | 17 | 0% | 13-62 -environmental_justice/__init__.py | 0 | 0 | 100% | -environmental_justice/admin.py | 5 | 0 | 100% | -environmental_justice/apps.py | 4 | 0 | 100% | -environmental_justice/models.py | 29 | 1 | 97% | 44 -environmental_justice/serializers.py | 6 | 0 | 100% | -environmental_justice/views.py | 23 | 0 | 100% | -feedback/__init__.py | 0 | 0 | 100% | -feedback/admin.py | 14 | 0 | 100% | -feedback/apps.py | 4 | 0 | 100% | -feedback/models.py | 42 | 15 | 64% | 20-29, 35-44, 61-63 -feedback/serializers.py | 10 | 0 | 100% | -feedback/urls.py | 4 | 0 | 100% | -feedback/views.py | 9 | 0 | 100% | -manage.py | 16 | 16 | 0% | 2-31 -merge_production_dotenvs_in_dotenv.py | 15 | 1 | 93% | 26 -scripts/ej/cmr_processing.py | 241 | 5 | 98% | 160, 186-188, 397, 410 -scripts/ej/config.py | 6 | 0 | 100% | -scripts/ej/test_cmr_processing.py | 225 | 1 | 99% | 610 -scripts/ej/test_threshold_processing.py | 97 | 1 | 99% | 209 -scripts/ej/threshold_processing.py | 20 | 0 | 100% | -sde_collections/__init__.py | 0 | 0 | 100% | -sde_collections/admin.py | 212 | 72 | 66% | 22-24, 29, 34, 40-60, 65-81, 86-89, 98-101, 110-112, 120-134, 143, 148, 153, 158, 163, 168, 173, 178-189, 196-197, 260, 265, 270, 275, 302-303, 308-309, 314-316, 345-372, 478-480 -sde_collections/apps.py | 4 | 0 | 100% | -sde_collections/forms.py | 15 | 0 | 100% | -sde_collections/management/commands/database_backup.py | 62 | 1 | 98% | 68 -sde_collections/management/commands/database_restore.py | 83 | 8 | 90% | 34, 36, 87-89, 142-145 -sde_collections/models/__init__.py | 0 | 0 | 100% | -sde_collections/models/candidate_url.py | 89 | 16 | 82% | 124, 128-134, 138-142, 145, 176-177 -sde_collections/models/collection.py | 414 | 144 | 65% | 241, 269, 277-287, 291-301, 305-315, 319-344, 348-357, 361, 365, 369-376, 380-387, 394, 403-406, 419, 436-439, 449-470, 478, 482-515, 519, 523, 527, 531-532, 536, 540-546, 550-553, 558-567, 575-617, 640, 679, 689, 703, 707-732, 765, 769-777, 785 -sde_collections/models/collection_choice_fields.py | 138 | 20 | 86% | 14-17, 36-39, 56-59, 74-77, 168-171 -sde_collections/models/delta_patterns.py | 313 | 33 | 89% | 119, 123, 139, 226-227, 263, 267, 291, 382-389, 439-449, 498, 503-506, 592, 627-641 -sde_collections/models/delta_url.py | 81 | 19 | 77% | 117-125, 129-135, 139-143, 146 -sde_collections/models/pattern.py | 145 | 79 | 46% | 40-48, 56-63, 66, 69, 73-74, 78-79, 87, 94-96, 105, 117-119, 128, 139-151, 163-205, 208-212, 215-216, 230-233, 243, 257-260, 268 -sde_collections/serializers.py | 191 | 47 | 75% | 80-81, 84-85, 88-89, 92-93, 129-130, 133-134, 137-138, 141-142, 197, 201, 211-214, 244-247, 257-260, 271, 274, 307-315, 335-343, 358-366 -sde_collections/sinequa_api.py | 102 | 3 | 97% | 65, 255, 289 -sde_collections/tasks.py | 119 | 67 | 44% | 25-67, 72-108, 113-117, 122-125, 130-148, 153-155, 215-216 -sde_collections/urls.py | 17 | 0 | 100% | -sde_collections/utils/__init__.py | 0 | 0 | 100% | -sde_collections/utils/bulk_github_push.py | 8 | 8 | 0% | 7-22 -sde_collections/utils/generate_deployment_message.py | 8 | 8 | 0% | 1-24 -sde_collections/utils/github_helper.py | 115 | 93 | 19% | 12-18, 30-42, 49-52, 60-68, 81-96, 104-110, 119-123, 127-129, 132-142, 145-152, 155-172, 175, 178-185, 189-192, 196-224, 227 -sde_collections/utils/health_check.py | 123 | 106 | 14% | 33-46, 51-57, 61-98, 102-143, 155-165, 172-187, 191-273 -sde_collections/utils/paired_field_descriptor.py | 33 | 2 | 94% | 35, 52 -sde_collections/utils/slack_utils.py | 19 | 4 | 79% | 57-58, 66-67 -sde_collections/utils/title_resolver.py | 90 | 5 | 94% | 64, 75, 83, 85, 92 -sde_collections/views.py | 368 | 229 | 38% | 70, 82-89, 102-141, 144-187, 194, 208-212, 215-223, 226-237, 246, 249-251, 256-265, 273-277, 280-306, 309-315, 323-327, 330-336, 339-345, 353-355, 358-368, 410, 413-422, 430, 433-442, 450, 458, 461-475, 483, 486-490, 505-511, 523-530, 538-566, 577-583, 586-607, 610-613, 628-634 -sde_indexing_helper/__init__.py | 2 | 0 | 100% | -sde_indexing_helper/conftest.py | 9 | 0 | 100% | -sde_indexing_helper/contrib/__init__.py | 0 | 0 | 100% | -sde_indexing_helper/contrib/sites/__init__.py | 0 | 0 | 100% | -sde_indexing_helper/users/__init__.py | 0 | 0 | 100% | -sde_indexing_helper/users/adapters.py | 11 | 11 | 0% | 1-16 -sde_indexing_helper/users/admin.py | 13 | 0 | 100% | -sde_indexing_helper/users/apps.py | 10 | 0 | 100% | -sde_indexing_helper/users/context_processors.py | 3 | 0 | 100% | -sde_indexing_helper/users/forms.py | 15 | 0 | 100% | -sde_indexing_helper/users/models.py | 10 | 0 | 100% | -sde_indexing_helper/users/tasks.py | 6 | 0 | 100% | -sde_indexing_helper/users/urls.py | 4 | 0 | 100% | -sde_indexing_helper/users/views.py | 27 | 0 | 100% | -sde_indexing_helper/utils/__init__.py | 0 | 0 | 100% | -sde_indexing_helper/utils/exceptions.py | 7 | 0 | 100% | -sde_indexing_helper/utils/storages.py | 7 | 7 | 0% | 1-11 -tests/test_merge_production_dotenvs_in_dotenv.py | 13 | 0 | 100 |% +COSMOS's tests grew up around the EJ portal, the URL lifecycle, and the pattern system — the parts +of the app that were always ours. The rewiring changed what sits at the edges: scraping is now +COSMOS → SSM → the crawl4ai crawler on EC2 → S3 → `DumpUrl`s, and indexing is now COSMOS → an S3 +export → `sts:AssumeRole` → `ecs:RunTask` → the WEB_COSMOS indexer, whose result COSMOS reads back +out of S3. Both of those halves were built with tests beside them. + +This document records where testing effort belongs and why, so that new tests land where a +regression is expensive rather than where code happens to be easy to exercise. It is deliberately +qualitative: the coverage numbers are produced by CI on every pull request, and a table pasted in +here goes stale the day after it is written. + +## How the suite runs +- `pytest.ini` pins `--ds=config.settings.test --reuse-db`; everything runs against + `config/settings/test.py`. +- CI (`.github/workflows/run_full_test_suite.yml`) triggers on pull requests to `dev`, builds the + `local.yml` stack, runs `bash ./init.sh`, then `coverage report`. +- `init.sh` runs each `test_*.py` file as its own pytest process under `coverage run --append`, + excluding `document_classifier/` and `functional_tests/`. To see current coverage locally: + `docker-compose -f local.yml run --rm django bash ./init.sh` followed by + `docker-compose -f local.yml run --rm django coverage report`. + +Two properties of the test settings are load-bearing and must survive any refactor of them: + +- `CELERY_BROKER_URL` is forced to `memory://` (both the setting and the environment variable). + Without it, any test that changes a workflow status would publish a real message to the same + Redis the local `celeryworker` consumes, and that worker would run the task against the *local* + database. +- Every pipeline setting (`SDE_S3_BUCKET`, `CRAWLER_INSTANCE_ID`, `SDE_INDEX_BUCKET`, `INDEXING_*`, + `SCRAPE_POLL_ENABLED`, `INDEX_POLL_ENABLED`, `INFERENCE_ENABLED`) defaults to blank/off, so a test + that forgets to mock is pointed at no real bucket, instance, or cluster. Tests that need those + values supply them with `override_settings`. + +## Where the tests live +| Location | What it covers | +|---|---| +| `sde_collections/tests/` | The bulk of the suite: scrape dispatch and ingest, the indexing hand-off, workflow-status triggers, the URL lifecycle, the pattern system, the inference flag, AWS session selection, the URL APIs, the backup/restore commands. | +| `sde_collections/tests/frontend/` | Selenium tests (auth, homepage features, pattern application). They `pytest.fail` unless `chromedriver` and `chromium` are on `PATH`. | +| `inference/tests/` | The classification pipeline, which is dormant (`INFERENCE_ENABLED` defaults to `False`) but not deleted. Its unit tests still run; `test_inference_integration.py` skips itself unless a live inference API is reachable. | +| `environmental_justice/tests/`, `sde_indexing_helper/users/tests/`, `scripts/ej/`, `tests/` | The EJ API, the user app, the EJ CMR/threshold processing scripts, and the dotenv merge helper. | +| `document_classifier/`, `functional_tests/` | Excluded from `init.sh` and from CI. `functional_tests/test_check_collection.py` is a Sinequa-era Selenium script pointed at the retired `sciencediscoveryengine.*` endpoints; it no longer describes anything the system does. | ## Critical Areas -### Config Generation -- config_generation/db_to_xml.py - - update_or_add_element_value() - - _update_config_xml() - - convert_template_to_scraper() - - add_document_type() - - add_url_exclude() - - add_title_mapping() - - add_job_list_item() - - get_tag_value() - - fetch_treeroot() - - fetch_document_type() -- config_generation/generate_jobs.py - - make_all_parallel_jobs() - -### Models - - environmental_justice/models.py - - sde_collections/models/collection.py - - clear_delta_urls() - - clear_dump_urls() - - refresh_url_lists_for_all_patterns () - - migrate_dump_to_delta () - - create_or_update_delta_url - - promote_to_curate - - add_to_public_query() - - create_scraper_config() - - create_indexer_config() - - create_plugin_config() - - _write_to_github() - - update_config_xml() - - apply_all_patterns() - - create_configs_on_status_change() - - sde_collections/models/collection_choice_fields.py - - sde_collections/models/delta_patterns.py - - sde_collections/models/delta_url.py - - sde_collections/models/pattern.py - - sde_indexing_helper/users/models.py - -### Views - - environmental_justice/views.py - - sde_collections/views.py - - sde_indexing_helper/users/views.py - -### Serializers and APIs - - environmental_justice/serializers.py - - sde_collections/serializers.py - -### Admin Interface - - environmental_justice/admin.py - - sde_collections/admin.py - - fetch_full_text_lrm_dev_action() - - fetch_full_text_xli_action() - - sde_indexing_helper/users/admin.py - -### Utilities and Helpers - - sde_collections/utils/github_helper.py - - sde_collections/utils/health_check.py - - sde_collections/utils/title_resolver.py - - sde_collections/utils/github_helper.py - - fetch_metadata() - - _get_contents_from_path() - -### Task Automation and Background Jobs - - sde_collections/tasks.py - -### Key Operational Pipelines in the Repository -The selection of critical areas for testing is guided by the following pipelines of the repository: -1. Sinequa config files are generated -2. COSMOS imports data from LRM Dev -3. Imported data is processed -4. Curators update URL metadata -5. Sinequa reads results from the COSMOS APIs - -### Critical Areas Lacking Tests -- **Config Generation**: Config generation files are under-tested. Develop unit tests for all critical functions in the config_generation files. -- **Project Settings**: Environment-specific configurations (`local.py`, `production.py`) have no tests. -- **Frontend Features**: Currently, there are no tests covering frontend logic and interactions. -- **Utilities and Helpers**: Essential utility modules like github_helper.py and health_check.py lack tests +### Scraping: dispatch and ingest +This is a contract with another repository, expressed in shell commands and S3 key names — the +class of thing that breaks silently. Cover, at minimum: + +- `sde_collections/scraping/job_builder.py` — `build_job_json()`: which override fields are emitted + (`None` is skipped, `False` is not), and the `MAX_PAGES_CAP` refusal. +- `sde_collections/scraping/ssm_dispatch.py` — `send_job_to_crawler()`: the exact command sent, the + `.tmp` + `mv` atomic delivery, shell quoting of hostile seed URLs, and the SSM comment limit. +- `sde_collections/scraping/s3_results.py` — the key layout, missing-key codes meaning "not + finished" rather than an error, and `results_ready()` rejecting a summary older than the latest + `ScrapeDispatch` (without which a re-dispatch would instantly "complete" against the previous + run's output). +- `sde_collections/tasks.py` — `dispatch_scrape_job()` (failure lands on **Scraping Failed** and + never raises), `poll_scrape_jobs()` (which statuses are scanned, the stall timeout), and + `ingest_scraped_collection()` (the compare-and-swap claim, zero documents counting as a failure, + replay idempotence, and failure after a claim not stranding the collection). + +### Indexing hand-off +The export layout is fixed by the indexer and cannot be renegotiated in a patch release, so its +shape belongs in tests: + +- `sde_collections/indexing/export.py` — manifest written **last**, `document_count` exactly + matching the JSONL line count, excluded URLs absent, title/label resolution, and the refusal to + export with a blank bucket. +- `sde_collections/indexing/dispatch.py` — the settings guard, role assumption, the full command + override (the image has no entrypoint, so the executable must be restated), and `RunTask` + failures surfacing as errors. +- `sde_collections/indexing/run_status.py` — status/validation reads under the `run_id` prefix, and + a missing `status.json` meaning "in flight". +- `sde_collections/tasks.py` — `index_collection_to_test()` / `index_collection_to_prod()` and + `poll_index_runs()`: the `IndexDispatch` record, the in-flight and failed statuses, prod success + mirroring the QC status the run entered with, unknown states counting as failure, and an old + run's status never resolving a newer dispatch. + +### Workflow status machine +`sde_collections/models/collection.py` is where the pipeline is actually wired: which status change +promotes, which enqueues an index run, the re-entrancy guard, the Slack notification (whose failure +must not break the save), and `WorkflowHistory` rows. Every status and reindexing status must +resolve to a button colour — an unmapped status silently rendering neutral is a real defect class. +The complementary assertion is negative: no removed Sinequa method may reappear on a transition. + +### URL lifecycle +`DumpUrl → DeltaUrl → CuratedUrl` is the core data model, described in +`sde_collections/models/README_LIFECYCLE.md`: + +- `Collection.migrate_dump_to_delta()` and `create_or_update_delta_url()` — the diff, including + deletion markers and `DELTA_COMPARISON_FIELDS`. +- `Collection.promote_to_curated()` — updates, deletions, metadata changes, repeated promotions, + and patterns re-applied afterwards. +- `sde_collections/models/delta_url.py`, `candidate_url.py`. + +### Pattern system +`sde_collections/models/delta_patterns.py` and `pattern.py` carry the most intricate logic in the +repo: apply and unapply for exclude, include, title, document type, division and other field +modifiers, plus specificity resolution when patterns overlap. This area is comparatively well +covered; keep it that way, and add a test for every reported mis-application rather than for the +fix alone. + +### Inference gating +The classification pipeline is disabled, not removed. What must stay tested is the gate itself: +`queue_necessary_classifications()` short-circuiting straight to migration when +`INFERENCE_ENABLED` is `False`, and `inference/signals.py` re-asserting `enabled` on its beat rows +from the flag on every `post_migrate` — the flag, not a hand-edit in the admin, is the source of +truth. Tests here should call the real method and patch only the queued task's `.delay`; patching +the method wholesale would guard nothing. + +### AWS session boundary +`sde_collections/utils/aws.py::get_boto3_session()` decides between explicit `SDE_AWS_*` keys (local +dev) and the default credential chain (instance role in AWS), and deliberately ignores the +`DJANGO_AWS_*` static-assets credentials. Partial keys must fall back rather than half-configure. + +### APIs, serializers, and the UI +`sde_collections/views.py` and `sde_collections/serializers.py` back the curation UI and the +DataTables endpoints. The list APIs keyed by `config_folder` are covered; the viewsets, the bulk +create path, and `CollectionDetailView.post` are thinner. + +### Operational surfaces +`sde_collections/management/commands/` (backup/restore are covered; the pipeline commands are not), +`sde_collections/admin.py` actions, and `sde_collections/utils/slack_utils.py` message formatting. + +## Critical Areas Lacking Tests +- **Beat-row creation for the pollers** — `sde_collections/signals.py` creates the `poll_scrape_jobs` + and `poll_index_runs` schedules on `post_migrate` and re-asserts `enabled` from + `SCRAPE_POLL_ENABLED` / `INDEX_POLL_ENABLED`. The equivalent handler in `inference/signals.py` is + tested; this one is not. +- **Pipeline management commands** — `dispatch_scrape`, `ingest_scrape_results`, + `migrate_urls_and_patterns`, `deduplicate_patterns`, `deduplicate_urls`, `export_urls_to_csv`, + `sync_with_production_webapp`. +- **Admin actions** — the CSV export, the exclude/include pattern actions, and the read-only + guarantees on the `ScrapeDispatch` / `IndexDispatch` admins. +- **Slack message construction** — `send_detailed_import_notification()` and + `send_indexing_validation_report()` are exercised only as patched call sites; the message bodies + themselves are untested. +- **Views and serializers** beyond the URL list APIs. +- **Project settings** — `config/settings/local.py` and `production.py` have no tests. +- **Frontend** — the Selenium suite covers auth, the homepage, and pattern application; the rest of + the curation UI is unverified, and the suite is skipped wherever Chromium is absent. + +## Conventions for new pipeline tests +- Mock at the seam, not at AWS: patch `get_boto3_session` *in the module under test* + (`sde_collections.indexing.export.get_boto3_session`, `…scraping.s3_results._get_object`), or the + task-module alias of a helper (`sde_collections.tasks.fetch_run_status`). Patching + `boto3` globally hides which client a module actually asks for. +- Drive the pipeline settings with `override_settings`; never rely on a developer's `.envs`. +- Patch `.delay` when a test changes a workflow status, unless the enqueue *is* what is being + asserted. +- Assert on the request that would have gone out — the SSM command text, the S3 key, the `RunTask` + overrides — rather than on the mock having been called. These are cross-repo contracts, and the + arguments are the contract. diff --git a/docs/documentation/sinequa_api.rst b/docs/documentation/sinequa_api.rst deleted file mode 100644 index 3f1569b2..00000000 --- a/docs/documentation/sinequa_api.rst +++ /dev/null @@ -1,12 +0,0 @@ - .. _sinequa_api: - -Using the Sinequa API -===================== -Overview ---------------------- -The Indexing Helper currently uses the API to start indexing arbitrary collections, and it is capable of executing queries as well. -The api class can be found in config_generation/api.py and by default is linked to the the ren server. - -The Sinequa documentation has several useful links. -`Generating an Access Token `_ - diff --git a/docs/howto.rst b/docs/howto.rst index fd6a13a7..183f9775 100644 --- a/docs/howto.rst +++ b/docs/howto.rst @@ -4,7 +4,7 @@ How To - Project Documentation Get Started ---------------------------------------------------------------------- -Documentation can be written as rst files in `sde_indexing_helper/docs`. +Documentation can be written as rst files in `docs/`. To build and serve docs, use the commands:: diff --git a/docs/index.rst b/docs/index.rst index 8faba818..9405d095 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,21 +1,20 @@ -.. SDE Indexing Helper documentation master file, created by +.. COSMOS documentation master file, created by sphinx-quickstart. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to SDE Indexing Helper's documentation! +Welcome to COSMOS's documentation! ====================================================================== .. toctree:: :maxdepth: 2 - :caption: Working with Sinequa: + :caption: Project Documentation: - documentation/sinequa_api howto .. toctree:: :maxdepth: 2 - :caption: Using the Indexing Helper: + :caption: Using COSMOS: documentation/indexing_guidelines @@ -24,5 +23,4 @@ Welcome to SDE Indexing Helper's documentation! Quick Links ================== -* :ref:`sinequa_api` * :ref:`indexing_guidelines` diff --git a/environmental_justice/tests/conftest.py b/environmental_justice/tests/conftest.py index d8b53c9a..df620f68 100644 --- a/environmental_justice/tests/conftest.py +++ b/environmental_justice/tests/conftest.py @@ -23,8 +23,10 @@ def client(): @pytest.fixture(autouse=True) -def setup_urls(): - """Setup URLs for testing""" - from django.conf import settings +def setup_urls(settings): + """Point ROOT_URLCONF at this module's router-only urlpatterns. + Uses pytest-django's `settings` fixture so the change is rolled back after each + test — assigning django.conf.settings directly leaked the EJ-only urlconf into + every later test module and broke all reverse() calls in a full-suite run.""" settings.ROOT_URLCONF = __name__ diff --git a/gitleaks-config.toml b/gitleaks-config.toml new file mode 100644 index 00000000..b65dab1c --- /dev/null +++ b/gitleaks-config.toml @@ -0,0 +1,18 @@ +# gitleaks configuration for the pre-commit hook (.pre-commit-config.yaml). +# Extends the built-in default ruleset; only allowlists are added here. +title = "COSMOS gitleaks config" + +[extend] +useDefault = true + +[allowlist] +description = "Paths that legitimately contain example/placeholder values" +paths = [ + '''\.env_sample$''', + '''\.envs/\.local/''', + '''gitleaks-config\.toml$''', +] +regexes = [ + # Placeholder AWS account used in tests + '''123456789012''', +] diff --git a/inference/inference_pipeline_queue.md b/inference/inference_pipeline_queue.md index d1c39a34..bea7f3e6 100644 --- a/inference/inference_pipeline_queue.md +++ b/inference/inference_pipeline_queue.md @@ -1,7 +1,19 @@ # COSMOS Inference Pipeline +> **DORMANT — the inference pipeline is disabled, not deleted.** +> As of Phase 2 the entire pipeline is gated on the `INFERENCE_ENABLED` setting +> (`config/settings/base.py`), which defaults to `False`. While the flag is off: +> - `Collection.queue_necessary_classifications()` never creates an `InferenceJob`; it +> goes straight to `migrate_dump_to_delta_and_handle_status_transistions`. +> - `process_inference_job_queue()` returns immediately without touching the queue. +> - The two `PeriodicTask` beat rows created in `inference/signals.py` are written with +> `enabled=settings.INFERENCE_ENABLED`, so beat never fires them. +> +> The models, tasks, and API client all remain in the tree. Everything below describes +> how the pipeline behaves **when `INFERENCE_ENABLED` is turned back on**. + ## Overview -The server runs both the COSMOS curation app and an ML Inference Pipeline, which can analyze and classify website content. COSMOS is process whole collections and send the full_texts of the individual urls to the Inference Pipeline for classification. Right now it supports Division Classifications and TDAMM Classifications. +The server runs both the COSMOS curation app and an ML Inference Pipeline, which can analyze and classify website content. When enabled, COSMOS processes whole collections and sends the full_texts of the individual urls to the Inference Pipeline for classification. Right now it supports Division Classifications and TDAMM Classifications. The Inference Pipeline can support multiple model versions for a single classification type. When a collection needs to be classified for certain classification and model, say "Division" and "v1", the COSMOS app will create an InferenceJob object. The InferenceJob will then create ExternalJob objects for each batch of urls in the collection. The ExternalJob objects will send the full_texts to the Inference Pipeline API, which will return a job_id. The ExternalJob will then ping the API with the job_id to get the results. Once all ExternalJobs are complete, the InferenceJob will be marked as complete. @@ -39,9 +51,12 @@ The inference pipeline uses a two-level job system: - InferenceJob is created for the collection/classification pair 2. **Chron** - - Every 5 minutes, between 6pm-7am, attempts to process_inference_job_queue() + - When enabled, every 5 minutes between 6pm-7am, attempts to process_inference_job_queue() - this could either mean batching and api sending - or it could mean reading in results from an open InferenceJob + - The beat rows are created disabled while `INFERENCE_ENABLED` is `False`, and + `process_inference_job_queue()` itself short-circuits on the same flag, so a + hand-enabled row or an ad-hoc invocation still processes nothing 3. **def process_inference_job_queue()** - Loop through all InferenceJob objects to find status=Pending diff --git a/inference/results_processing.md b/inference/results_processing.md index 4a47e412..d26ec0cd 100644 --- a/inference/results_processing.md +++ b/inference/results_processing.md @@ -1,3 +1,10 @@ +> **DORMANT — the inference pipeline is disabled, not deleted.** +> As of Phase 2 classification is gated on the `INFERENCE_ENABLED` setting +> (`config/settings/base.py`), which defaults to `False`. With the flag off, +> `Collection.queue_necessary_classifications()` skips job creation entirely and +> DumpUrls migrate straight to DeltaUrls, so no classification stage runs between them. +> The design below describes the DumpUrl classification stage **when +> `INFERENCE_ENABLED` is turned back on**. ## Classifying Collections @@ -43,7 +50,7 @@ Pros - By using the DumpUrl and the associated promotion code, we can piggy back on the DeltaUrl determination processes to handle delta generation Cons -- You have to re-pull from dev in order to classify +- You have to re-scrape the collection in order to classify - Promotion has to wait on inference server processing (this is also a pro, as Emily will never see until the processing is done) ### Dedicated Process diff --git a/inference/signals.py b/inference/signals.py index 89e83f70..337f8491 100644 --- a/inference/signals.py +++ b/inference/signals.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.db.models.signals import post_migrate from django.dispatch import receiver @@ -34,11 +35,15 @@ def create_periodic_tasks(sender, **kwargs): crontab=weekday_crontab, name=weekday_task_name, task="inference.tasks.process_inference_job_queue", + enabled=settings.INFERENCE_ENABLED, ) else: weekday_task = PeriodicTask.objects.get(name=weekday_task_name) weekday_task.crontab = weekday_crontab weekday_task.task = "inference.tasks.process_inference_job_queue" + # Re-asserted on every migrate so the flag, not a hand-edit in the admin, + # is the source of truth — a manual disable would not survive a deploy. + weekday_task.enabled = settings.INFERENCE_ENABLED weekday_task.save() # Check if weekend task exists @@ -50,9 +55,11 @@ def create_periodic_tasks(sender, **kwargs): crontab=weekend_crontab, name=weekend_task_name, task="inference.tasks.process_inference_job_queue", + enabled=settings.INFERENCE_ENABLED, ) else: weekend_task = PeriodicTask.objects.get(name=weekend_task_name) weekend_task.crontab = weekend_crontab weekend_task.task = "inference.tasks.process_inference_job_queue" + weekend_task.enabled = settings.INFERENCE_ENABLED weekend_task.save() diff --git a/inference/tasks.py b/inference/tasks.py index cba5b43f..92334dff 100644 --- a/inference/tasks.py +++ b/inference/tasks.py @@ -1,5 +1,6 @@ # inference/tasks.py from celery import shared_task +from django.conf import settings from inference.models import InferenceJob, InferenceJobStatus from inference.utils.advisory_lock import AdvisoryLock @@ -11,6 +12,11 @@ def process_inference_job_queue(): Main job queue processor that runs every 5 minutes between 6pm-7am. Uses Postgres advisory locking to ensure only one instance runs at a time. """ + # Belt and braces: the beat rows are disabled when the flag is off, but a manually + # enabled row or ad-hoc invocation must not process the queue either. + if not settings.INFERENCE_ENABLED: + return "Inference pipeline disabled (INFERENCE_ENABLED=False)" + lock = AdvisoryLock("inference_queue_lock") with lock.hold() as acquired: diff --git a/inference/tests/test_batch.py b/inference/tests/test_batch.py index b6d10963..9578b15d 100644 --- a/inference/tests/test_batch.py +++ b/inference/tests/test_batch.py @@ -1,6 +1,6 @@ # inference/tests/test_batch.py # docker-compose -f local.yml run --rm django pytest inference/tests/test_batch.py -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock import pytest from django.db.models import QuerySet @@ -307,31 +307,24 @@ def close(self): # Verify close was still called despite the exception mock_iterator.close.assert_called_once() - def test_integration_with_mock_django_db(self, processor): - """Test integration with mocked Django DB objects""" + def test_integration_with_factory_urls(self, processor): + """Real model instances (unsaved factory builds) flow through prepare_url_data.""" from sde_collections.tests.factories import DumpUrlFactory - # Create a patch for the QuerySet iterator that returns our factory objects - with patch.object(QuerySet, "iterator") as mock_iterator: - # Create mock URLs using the factory - url1 = DumpUrlFactory.build(id=1, scraped_text="Text 1") - url2 = DumpUrlFactory.build(id=2, scraped_text="Text 2") + url1 = DumpUrlFactory.build(id=1, scraped_text="Text 1") + url2 = DumpUrlFactory.build(id=2, scraped_text="Text 2") - # Set up the mock to return these objects - mock_iterator.return_value = iter([url1, url2]) - - # Create a real QuerySet (that will use our mock iterator) - mock_queryset = MagicMock(spec=QuerySet) - mock_queryset.iterator = mock_iterator + mock_queryset = MagicMock(spec=QuerySet) + mock_queryset.iterator.return_value = iter([url1, url2]) - # Process the batches - batches = list(processor.iter_url_batches(mock_queryset)) + batches = list(processor.iter_url_batches(mock_queryset)) - # Verify the output - assert len(batches) == 1 # Both URLs fit in one batch - assert len(batches[0]) == 2 - assert batches[0][0]["url_id"] == 1 - assert batches[0][1]["url_id"] == 2 + assert len(batches) == 1 # Both URLs fit in one batch + assert len(batches[0]) == 2 + assert batches[0][0]["url_id"] == 1 + assert batches[0][0]["text"] == "Text 1" + assert batches[0][0]["metadata"] == {"title": url1.scraped_title or "", "url": url1.url} + assert batches[0][1]["url_id"] == 2 # Additional test class to identify potential issues @@ -339,30 +332,25 @@ class TestBatchProcessorPotentialIssues: """Tests focused on identifying potential problems with BatchProcessor""" def test_extremely_large_text(self): - """Test handling of extremely large text to check for memory issues""" - processor = BatchProcessor() - - # Create a URL with extremely large text (100MB) - # This is simulated rather than actually creating such a large string - large_text_size = 100 * 1024 * 1024 # 100MB + """An oversized URL must be truncated to the batch limit and yielded as its own + batch — with real text against a small limit, not by patching the processor.""" + processor = BatchProcessor(max_batch_text_length=100) url = Mock() url.id = 1 + url.scraped_text = "A" * 250 + url.scraped_title = "Big page" + url.url = "https://example.com/big" - # Instead of creating a huge string, we'll patch get_text_length - # to simulate the size calculation - with patch.object(processor, "get_text_length", return_value=large_text_size): - url_data = {"url_id": url.id, "text": "LARGE", "metadata": {}} - - with patch.object(processor, "prepare_url_data", return_value=url_data): - mock_queryset = MagicMock(spec=QuerySet) - mock_queryset.iterator.return_value = iter([url]) + mock_queryset = MagicMock(spec=QuerySet) + mock_queryset.iterator.return_value = iter([url]) - batches = list(processor.iter_url_batches(mock_queryset)) + batches = list(processor.iter_url_batches(mock_queryset)) - # Should create a single batch with truncated content - assert len(batches) == 1 - assert len(batches[0]) == 1 + assert len(batches) == 1 + assert len(batches[0]) == 1 + assert batches[0][0]["text"] == "A" * 100 # actually truncated to the limit + assert batches[0][0]["url_id"] == 1 def test_url_with_no_text(self): """Test handling of URLs with empty text""" diff --git a/inference/tests/test_classification_utils.py b/inference/tests/test_classification_utils.py index 2f992f3d..230347bc 100644 --- a/inference/tests/test_classification_utils.py +++ b/inference/tests/test_classification_utils.py @@ -153,7 +153,7 @@ def test_update_url_properly_calls_mapping(self, mock_map_function, mock_url): result = update_url_with_classification_results(mock_url, classification_results) # Verify map_classification_to_tdamm_tags was called properly - mock_map_function.assert_called_once_with(classification_results) + mock_map_function.assert_called_once_with(classification_results, threshold=None) # Verify URL object was updated correctly assert mock_url.tdamm_tag_ml == mock_tdamm_tags @@ -164,7 +164,8 @@ def test_update_url_properly_calls_mapping(self, mock_map_function, mock_url): @patch("inference.utils.classification_utils.map_classification_to_tdamm_tags") def test_threshold_parameter_behavior(self, mock_map_function, mock_url): - """Test how threshold parameter is handled""" + """A caller-supplied threshold must reach the mapping function (it used to be + silently discarded, so every caller got settings.TDAMM_CLASSIFICATION_THRESHOLD).""" mock_tdamm_tags = ["MMA_M_EM_O"] mock_map_function.return_value = mock_tdamm_tags @@ -173,8 +174,7 @@ def test_threshold_parameter_behavior(self, mock_map_function, mock_url): update_url_with_classification_results(mock_url, classification_results, threshold=custom_threshold) - # Based on the implementation, the function doesn't pass the threshold parameter - mock_map_function.assert_called_once_with(classification_results) + mock_map_function.assert_called_once_with(classification_results, threshold=custom_threshold) def test_integration_with_real_mapping(self, mock_url): """Test end-to-end integration with real mapping function""" diff --git a/inference/utils/classification_utils.py b/inference/utils/classification_utils.py index 0e757e36..61915c97 100644 --- a/inference/utils/classification_utils.py +++ b/inference/utils/classification_utils.py @@ -82,7 +82,7 @@ def update_url_with_classification_results(url_object, classification_results, t Returns: list: The list of TDAMM tags that were applied """ - tdamm_tags = map_classification_to_tdamm_tags(classification_results) + tdamm_tags = map_classification_to_tdamm_tags(classification_results, threshold=threshold) # Update the URL object url_object.tdamm_tag_ml = tdamm_tags diff --git a/local.yml b/local.yml index b52a4950..19788bfb 100644 --- a/local.yml +++ b/local.yml @@ -18,6 +18,9 @@ services: env_file: - ./.envs/.local/.django - ./.envs/.local/.postgres + # optional, gitignored: local-only AWS creds for the SDE pipeline (overrides .django) + - path: ./.envs/.local/.sde-aws + required: false ports: - "8001:8000" # this prevents conflicts with inference pipeline command: /start diff --git a/requirements/base.txt b/requirements/base.txt index a4ca9cf8..04112c90 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -27,10 +27,8 @@ djangorestframework-datatables==0.7.2 djangorestframework==3.15.2 factory-boy==3.3.0 lxml==4.9.2 -PyGithub==2.2.0 pytest-django==4.8.0 pytest==8.0.0 tenacity==8.2.2 tqdm==4.66.3 unidecode==1.3.8 -xmltodict==0.13.0 diff --git a/requirements/local.txt b/requirements/local.txt index 6db73be1..895ab7dd 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -12,7 +12,6 @@ django-stubs==4.2.7 # https://github.com/typeddjango/django-stubs pytest==8.0.0 # https://github.com/pytest-dev/pytest pytest-sugar==1.0.0 # https://github.com/Frozenball/pytest-sugar types-requests # maybe instead, we should add `mypy --install-types` to the dockerfile? -types-xmltodict pytest-xdist>=3.3.1 pytest-cov>=4.1.0 selenium>=4.15.2 # Selenium (Frontend Testing) diff --git a/scripts/ej/README.md b/scripts/ej/README.md index eb74c490..fee33db4 100644 --- a/scripts/ej/README.md +++ b/scripts/ej/README.md @@ -70,7 +70,7 @@ The pipeline generates a JSON file named `ej_dump_YYYYMMDD_HHMMSS.json` containi To deploy the output to the server: ```bash # Copy to server -scp ej_dump_YYYYMMDD_HHMMSS.json sde:/home/ec2-user/sde_indexing_helper/backups/ +scp ej_dump_YYYYMMDD_HHMMSS.json sde:/home/ec2-user/sde-indexing-helper/backups/ # Process on server using dm shell dmshell diff --git a/scripts/health_checks_on_urls_titles.py b/scripts/health_checks_on_urls_titles.py deleted file mode 100644 index 60387c3c..00000000 --- a/scripts/health_checks_on_urls_titles.py +++ /dev/null @@ -1,58 +0,0 @@ -from sde_collections.models.candidate_url import CandidateURL -from sde_collections.models.collection import Collection -from sde_collections.sinequa_api import Api - - -def _health_check_on_urls_titles(server_name: str): - if server_name == "test": - url_field = "download_url" - status_field = "present_on_test" - title_field = "test_title" - elif server_name == "production": - url_field = "url1" - status_field = "present_on_prod" - title_field = "production_title" - else: - # Handle invalid server name - raise ValueError(f"Invalid server name: {server_name}") - - api = Api(server_name=server_name) - - collection_config_folders = [collection.config_folder for collection in Collection.objects.all()] - - for collection_config_folder in collection_config_folders: - page = 1 - urls_server_info_dict = {} - while True: - response = api.query(page=page, collection_config_folder=collection_config_folder) - if response.get("cursorRowCount", 0) == 0: # Safeguard against missing 'cursorRowCount' - break - for record in response.get("records", []): # Safeguard against missing 'records' - url = record.get(url_field) - title = record.get("title") - if url and title: # Ensure both url and title are present - urls_server_info_dict[url] = {"title": title} - page += 1 - print(f"Finished collecting URLs from {server_name} server for config folder {collection_config_folder}") - - collection_object = Collection.objects.filter(config_folder=collection_config_folder) - candidate_urls_objects = CandidateURL.objects.filter(collection=collection_object[0]) - for candidate_urls_object in candidate_urls_objects: - is_present_on_server = candidate_urls_object.url in urls_server_info_dict.keys() - if getattr(candidate_urls_object, status_field) != is_present_on_server: - setattr(candidate_urls_object, status_field, is_present_on_server) - try: - setattr( - candidate_urls_object, - title_field, - urls_server_info_dict.get(candidate_urls_object.url)["title"], - ) - except TypeError: - setattr(candidate_urls_object, title_field, "Unavailable") - candidate_urls_object.save() - print(f"Finished updating urls within collection config folder {collection_config_folder}") - - -if __name__ == "__main__": - _health_check_on_urls_titles(server_name="test") - _health_check_on_urls_titles(server_name="production") diff --git a/scripts/push_curated_collections_to_github.py b/scripts/push_curated_collections_to_github.py deleted file mode 100644 index f646c4c1..00000000 --- a/scripts/push_curated_collections_to_github.py +++ /dev/null @@ -1,17 +0,0 @@ -from sde_collections.models.collection import Collection -from sde_collections.models.collection_choice_fields import WorkflowStatusChoices -from sde_collections.utils.github_helper import GitHubHandler - - -def push_curated_collections_to_github(): - # Filter collections with a specific workflow status (CURATED) - collections = Collection.objects.filter(workflow_status=WorkflowStatusChoices.CURATED) - - # Initialize the GitHub handler and push collections - github_handler = GitHubHandler(collections) - github_handler.push_to_github() - print("Curated collections with a workflow status of CURATED have been pushed to GitHub.") - - -if __name__ == "__main__": - push_curated_collections_to_github() diff --git a/scripts/quality_and_indexing/find_missing_folders.py b/scripts/quality_and_indexing/find_missing_folders.py deleted file mode 100644 index 8ebd900b..00000000 --- a/scripts/quality_and_indexing/find_missing_folders.py +++ /dev/null @@ -1,60 +0,0 @@ -import os - -from sde_collections.models.collection import Collection -from sde_collections.models.collection_choice_fields import WorkflowStatusChoices -from sde_collections.utils.github_helper import GitHubHandler - - -def get_sources_by_status(statuses): - """Fetch sources by workflow status.""" - return Collection.objects.filter(workflow_status__in=statuses) - - -def get_missing_folders(collections, base_directory, github_handler): - """Find collections missing specific folders in the base directory.""" - missing = [] - for source in collections: - folder_path = os.path.join(base_directory, source.config_folder, "default.xml") - if not github_handler.check_file_exists(folder_path): - missing.append(source) - return missing - - -def get_difference(queryset, *exclude_lists): - """Return queryset minus elements in exclude_lists based on config_folder.""" - exclude_folders = {item.config_folder for sublist in exclude_lists for item in sublist} - return [item for item in queryset if item.config_folder not in exclude_folders] - - -def print_configs(queryset): - """Print the config folder paths of the collections in the queryset.""" - for source in queryset: - print(source.config_folder) - print("---" * 20) - print() - - -# initial sources list -print("sources_to_fix") -print_configs(get_sources_by_status([WorkflowStatusChoices.QUALITY_FIXED])) - -print("sources_to_curated") -print_configs(get_sources_by_status([WorkflowStatusChoices.CURATED])) - -all_relevant_sources = get_sources_by_status([WorkflowStatusChoices.QUALITY_FIXED, WorkflowStatusChoices.CURATED]) - -# broken sources list -github_handler = GitHubHandler() -print("missing_scraper_folders") -missing_scraper_folders = get_missing_folders(all_relevant_sources, "sources/scrapers/", github_handler) -print_configs(missing_scraper_folders) - -print("missing_plugin_folders") -missing_plugin_folders = get_missing_folders(all_relevant_sources, "sources/SDE/", github_handler) -print_configs(missing_plugin_folders) - - -# final sources list -final_sources = get_difference(all_relevant_sources, missing_scraper_folders, missing_plugin_folders) -print("final_sources") -print_configs(final_sources) diff --git a/scripts/update_has_sinequa_config.py b/scripts/update_has_sinequa_config.py deleted file mode 100644 index b40cc93c..00000000 --- a/scripts/update_has_sinequa_config.py +++ /dev/null @@ -1,7 +0,0 @@ -from sde_collections.models.collection import Collection - -with open("no_folders.txt") as no_folders_list: - no_folders = no_folders_list.readlines() - -for no_folder in no_folders: - Collection.objects.filter(name=no_folder.strip()).update(has_sinequa_config=False) diff --git a/sde_collections/DEPLOYMENT.md b/sde_collections/DEPLOYMENT.md new file mode 100644 index 00000000..4923c8d3 --- /dev/null +++ b/sde_collections/DEPLOYMENT.md @@ -0,0 +1,190 @@ +# COSMOS Deployment and CI/CD + +> Companion to [WORKFLOW.md](../WORKFLOW.md) (the curation pipeline). +> Database backup and restore are covered in [SQLDumpRestoration.md](../SQLDumpRestoration.md). + +**Deploys are manual today.** Everything under "Current state" describes the repo as it actually +is; everything under "Proposed pipeline" is a target that does **not** exist yet — there is no +deploy script and no deployment workflow in `.github/workflows/`. Keep the two sections distinct. + +--- + +## Current state + +| Area | What exists today | +|---|---| +| Deploy mechanism | Manual: `ssh` to the host, then rebuild in place. No deploy script, no image registry, no automation in the repo. | +| Branches | `dev`, `staging`, `production` all exist on `origin`. | +| CI | One workflow, `.github/workflows/run_full_test_suite.yml`, triggered **only** on PRs into `dev` (`paths-ignore: '**/*.md'`). | +| CI test runner | `init.sh` — finds every `test_*.py` and runs each in its **own** `coverage run --append -m pytest` process, counting failures. Excludes `document_classifier` and `functional_tests`. | +| Other workflows | `.github/workflows/issue-formatter.yml` (issue body templating). Nothing else. | +| Pre-commit | `.pre-commit-config.yaml` with black, isort, flake8, pyupgrade, bandit, mypy (excluded), and gitleaks. `pre-commit.ci` is enabled with weekly autoupdate. | +| Compose stacks | `local.yml` and `production.yml`. Four services share the Django image: `django`, `celeryworker`, `celerybeat`, `flower`. `production.yml` adds `traefik`, `postgres`, `awscli`. | +| Backup tooling | `manage.py database_backup` and `manage.py database_restore` exist in `sde_collections/management/commands/`. | +| Beat schedules | No `CELERY_BEAT_SCHEDULE` setting anywhere. All schedules are `django_celery_beat` **database rows**, written by `post_migrate` receivers in `inference/signals.py` and `sde_collections/signals.py`. The latter writes "Poll crawler S3 results (every 5 min)" and "Poll index runs (every 2 min)", each enabled from `SCRAPE_POLL_ENABLED` / `INDEX_POLL_ENABLED`. Because `enabled` is re-asserted at migrate time, **changing one of those flags requires `manage.py migrate`** — restarting the services alone does nothing. | +| Credentials | Two separate scopes. django-storages static assets use static keys (`DJANGO_AWS_ACCESS_KEY_ID` / `DJANGO_AWS_SECRET_ACCESS_KEY`). The pipeline uses `sde_collections/utils/aws.py::get_boto3_session()`, which deliberately does *not* read those: it prefers `SDE_AWS_*` if set and otherwise falls through to the default chain (the host's instance role). Dispatching an index run additionally requires `sts:AssumeRole` on the indexer's dispatch role. Env files (`.envs/.production/.django`) are maintained by hand on each host. | +| Health endpoint | **None.** There is no `/healthz` or equivalent. | + +### Two defects worth fixing regardless of CI/CD + +1. **The gitleaks hook cannot run.** `.pre-commit-config.yaml` passes + `--config=gitleaks-config.toml`, but that file does not exist and is not tracked. The hook + fails rather than scanning. Either add the config or drop the argument to use gitleaks' defaults. +2. **A live production credential is committed.** `SQLDumpRestoration.md` contains the production + Postgres password and the RDS endpoint hostname in the current `HEAD` (lines 101 and 117) — not + only in history. **Rotate the credential, then scrub the file.** Rotation is the part that + matters; history rewriting is optional once the secret is dead. + +--- + +## Why automate this + +Three reasons, in order of weight: + +1. **The release path is untested.** CI runs only on PRs into `dev`. Merges into `staging` and + `production` are exercised by nothing, so the branches that actually ship are the least verified. +2. **Manual deploys are not reversible.** `ssh` + rebuild-on-host leaves no artifact to roll back + to and no record of what is running. Recovery means rebuilding from a guess about the last-good + commit. +3. **The rewiring adds automated side effects.** Once [WORKFLOW.md](../WORKFLOW.md) lands, a + workflow-status change dispatches an SSM command to a live EC2 crawler and triggers a Celery + task that deletes and rebuilds a collection's `DumpUrl` rows. A bad deploy stops being a broken + UI and becomes unwanted writes to shared infrastructure. + +> The indexing hand-off now exists. COSMOS exports curated URLs to S3, assumes an IAM role in the +> indexer's account, and calls `ecs:RunTask`; the indexer writes to OpenSearch. The settings that +> gate it — `SDE_INDEX_BUCKET`, `INDEXING_ECS_CLUSTER`, `INDEXING_TASK_FAMILY`, +> `INDEXING_DISPATCH_ROLE_ARN`, `INDEXING_SUBNETS`, `INDEXING_SECURITY_GROUPS`, `INDEX_POLL_ENABLED` +> — all default blank or off, so a host that has not been wired cannot dispatch. `validate_deploy_env` +> and `preflight_aws` should cover them: the bucket's reachability and the `sts:AssumeRole` grant on +> the dispatch role. + +--- + +## Proposed pipeline + +### Branch and environment model + +``` +feature branch ──PR──► dev ──PR──► staging ──PR──► production + │ │ │ + │ │ └─► prod host (manual approval) + │ └───────────────────► staging host (automatic) + └─ CI only +``` + +CI should run on pull requests into **all three** branches, closing the gap in point 1 above. + +### Workflows + +| Workflow | Trigger | Jobs | +|---|---|---| +| `ci.yml` | PR into `dev`, `staging`, `production` | `run-tests`, `django-checks` (`check --deploy`, `makemigrations --check --dry-run`) | +| `deploy-staging.yml` | push to `staging` | build → ECR → SSM → `scripts/deploy.sh --environment staging` | +| `deploy-production.yml` | push to `production` | staging-digest check → SSM → `scripts/deploy.sh --environment production` | +| `rollback.yml` | manual dispatch | redeploy a named image tag to either environment | +| `secret-scan-history.yml` | weekly + manual | full-history gitleaks, report-only | + +`ci.yml` replaces `run_full_test_suite.yml`. Take the opportunity to drop the per-file loop in +`init.sh` and run `pytest` once — the loop spawns a process per test file and re-pays Django setup +each time, for no isolation benefit that `--reuse-db` does not already provide. + +All `deploy-*` and `rollback` workflows gate on repository variable **`CD_ENABLED`**. Until it is +`true`, they skip. This lets the workflows merge and be reviewed before they can touch a host. + +### What a deploy does + +`scripts/deploy.sh` runs on the host and is the single definition of a deploy — GitHub Actions only +decides *when* to call it and *with which tag*. Order matters: + +1. **Fetch the artifact** — pull the image from ECR (`--image-tag`), or check out and rebuild on the + host (`--git-ref`, the fallback mode that needs no registry). +2. **Validate the environment** (`manage.py validate_deploy_env`) — *before* anything mutates. + A host missing required settings must fail here, not inside a Celery task that has already + half-completed its work. +3. **Back up** — `manage.py database_backup` plus an RDS snapshot. Production only. +4. **Migrate** — `manage.py migrate --noinput`. +5. **Swap containers** — `docker compose up -d`, recreating all four services that share the Django + image. **`celerybeat` especially:** beat schedules are database rows written by `post_migrate`, + and a beat process left running on the old schedule silently ignores them. +6. **Smoke checks** — `check --deploy`, `celery inspect ping`, an assertion that the expected + `PeriodicTask` rows exist with the expected `enabled` state, and `manage.py preflight_aws`. +7. **On failure** — redeploy the previously recorded tag and post to Slack. + +### Two new management commands + +Both belong in `sde_collections/management/commands/`, alongside the existing backup commands. + +- **`validate_deploy_env`** — fails fast when required settings are missing or contradictory. + Once test and prod search endpoints exist, it must refuse a host where the two are identical; + that misconfiguration would make the QC gate decorative while appearing to pass. +- **`preflight_aws`** — checks AWS reachability from the deployed host's own credentials + (SSM to the crawler instance, S3 read on the crawler bucket, and later the search and embedding + endpoints). Running it on every deploy stops it being a step someone remembers to do. It should + report every check independently rather than aborting on the first failure. + +### Prerequisite: a health endpoint + +Smoke checks need one and the repo has none. Add a minimal `/healthz` view returning 200 plus a +database connectivity check. Keep it unauthenticated and cheap enough for a load balancer. + +--- + +## Rollback + +```bash +gh workflow run rollback.yml \ + -f environment=production \ + -f image_tag=sha- \ + -f git_sha= +``` + +Migrations are **not** reversed. That is safe only while every migration is additive and +backward-compatible — old code ignoring new columns and new tables. The rewiring's migrations are +designed to hold that property, so rolling the image back is a complete rollback. + +If a future migration breaks it (drops a column, renumbers an enum, backfills destructively), the PR +introducing it must say so, and rollback for that release becomes restore-from-backup instead. + +--- + +## Prerequisites before enabling CD + +| Item | Where | +|---|---| +| ECR repository | SMCE account, `us-east-1` | +| GitHub OIDC → IAM deploy role | secret `AWS_DEPLOY_ROLE_ARN` | +| SSM agent + instance role on both COSMOS hosts | verify with `aws ssm describe-instance-information` | +| Repo variables | `CD_ENABLED`, `AWS_REGION`, `ECR_REPOSITORY`, `STAGING_INSTANCE_ID`, `PRODUCTION_INSTANCE_ID`, `PRODUCTION_RDS_INSTANCE_ID` | +| GitHub Environment `production` | required reviewers configured | +| Branch protection | required checks on `dev`, `staging`, `production` (needs a repo admin) | + +The **deploy role** needs `ecr:*` on the repository plus `ssm:SendCommand` and +`ssm:GetCommandInvocation` scoped to the two instances. The **hosts' own roles** need whatever the +curation pipeline uses: SSM to the crawler instance, S3 read on the crawler bucket, read/write on +the indexing bucket (`SDE_INDEX_BUCKET`), and `sts:AssumeRole` on the indexer's dispatch role. + +Verify before flipping `CD_ENABLED`: + +```bash +aws ssm describe-instance-information \ + --filters "Key=InstanceIds,Values=" \ + --query 'InstanceInformationList[0].PingStatus' --output text # expect: Online +aws ecr describe-repositories --repository-names cosmos # expect: no error +``` + +Then rehearse a rollback **on staging** before trusting it on production. + +--- + +## Open questions + +- **Credential model.** The codebase passes static AWS keys; this design assumes host instance + roles. Pick one. Instance roles are the better target, but the migration has to be deliberate + rather than a side effect of a deploy change. +- **Env-file source of truth.** `.envs/.production/.django` is hand-maintained per host, so + `validate_deploy_env` can only check whatever that host happens to hold. Moving it to SSM + Parameter Store or Secrets Manager and rendering it at deploy time would make the check meaningful. +- **Build-on-host vs registry.** The fallback `--git-ref` mode avoids standing up ECR entirely. + If ECR is slow to obtain, shipping the fallback first is a viable staged path — it still gives a + scripted, validated, reversible deploy. diff --git a/sde_collections/admin.py b/sde_collections/admin.py index 766d8f3f..7025c564 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -14,50 +14,9 @@ from .models.collection import Collection, ReindexingHistory, WorkflowHistory from .models.collection_choice_fields import TDAMMTags from .models.delta_url import CuratedUrl, DeltaUrl, DumpUrl +from .models.indexing import IndexDispatch from .models.pattern import DivisionPattern, IncludePattern, TitlePattern -from .tasks import fetch_full_text, import_candidate_urls_from_api - - -def fetch_and_replace_text_for_server(modeladmin, request, queryset, server_name): - for collection in queryset: - fetch_full_text.delay(collection.id, server_name) - modeladmin.message_user(request, f"Started importing URLs from {server_name.upper()} Server") - - -@admin.action(description="Import candidate URLs from LRM Dev Server with Full Text") -def fetch_full_text_lrm_dev_action(modeladmin, request, queryset): - fetch_and_replace_text_for_server(modeladmin, request, queryset, "lrm_dev") - - -@admin.action(description="Import candidate URLs from XLI Server with Full Text") -def fetch_full_text_xli_action(modeladmin, request, queryset): - fetch_and_replace_text_for_server(modeladmin, request, queryset, "xli") - - -@admin.action(description="Generate deployment message") -def generate_deployment_message(modeladmin, request, queryset): - # generate deployment message - response = HttpResponse(content_type="text/txt") - - message_start = """:rocket: Production Deployment Update :rocket: -Hello Team, - -I'm pleased to announce that we have successfully moved several key collections -to our production environment as part of our latest deployment! :tada:\n -Collections Now Live in Prod:\n""" - - message_middle = "\n\n".join( - [f"- {collection.name} | {collection.server_url_prod}" for collection in queryset.all()] - ) - - message_end = """ -If you find something needs changing, please let us know. - -Dev Team""" - - response.content = f"{message_start}\n{message_middle}\n{message_end}" - - return response +from .models.scraper_config import ScrapeDispatch, ScraperConfigOverride @admin.action(description="Download candidate URLs as csv") @@ -81,30 +40,6 @@ def download_candidate_urls_as_csv(modeladmin, request, queryset): return response -@admin.action(description="Import metadata from Sinequa configs") -def import_sinequa_metadata(modeladmin, request, queryset): - for collection in queryset.all(): - # eventually this needs to be done in celery - collection.import_metadata_from_sinequa_config() - messages.add_message( - request, - messages.INFO, - f"Imported metadata for collection: {collection.name}", - ) - - -@admin.action(description="Export metadata to Sinequa config") -def export_sinequa_metadata(modeladmin, request, queryset): - for collection in queryset.all(): - # eventually this needs to be done in celery - collection.export_metadata_to_sinequa_config() - messages.add_message( - request, - messages.INFO, - f"Exported sinequa config for collection: {collection.name}", - ) - - @admin.action(description="Generate candidate URLs") def generate_candidate_urls(modeladmin, request, queryset): collection = queryset.first() @@ -116,63 +51,6 @@ def generate_candidate_urls(modeladmin, request, queryset): ) -def import_candidate_urls_from_api_caller(modeladmin, request, queryset, server_name): - id_list = queryset.values_list("id", flat=True) - if len(id_list) > 1: - messages.add_message( - request, - messages.ERROR, - "We can only import one collection at a time using the admin action." - " Consider using the django shell for bulk imports.", - ) - return - import_candidate_urls_from_api.delay( - collection_ids=list(queryset.values_list("id", flat=True)), - server_name=server_name, - ) - collection_names = ", ".join(queryset.values_list("name", flat=True)) - messages.add_message( - request, - messages.INFO, - f"Started importing URLs from the API for: {collection_names} from {server_name.upper()} Server", - ) - - -@admin.action(description="Import candidate URLs from Test") -def import_candidate_urls_test(modeladmin, request, queryset): - import_candidate_urls_from_api_caller(modeladmin, request, queryset, "test") - - -@admin.action(description="Import candidate URLs from Production") -def import_candidate_urls_production(modeladmin, request, queryset): - import_candidate_urls_from_api_caller(modeladmin, request, queryset, "production") - - -@admin.action(description="Import candidate URLs from Secret Test") -def import_candidate_urls_secret_test(modeladmin, request, queryset): - import_candidate_urls_from_api_caller(modeladmin, request, queryset, "secret_test") - - -@admin.action(description="Import candidate URLs from Secret Production") -def import_candidate_urls_secret_production(modeladmin, request, queryset): - import_candidate_urls_from_api_caller(modeladmin, request, queryset, "secret_production") - - -@admin.action(description="Import candidate URLs from XLI Server") -def import_candidate_urls_xli_server(modeladmin, request, queryset): - import_candidate_urls_from_api_caller(modeladmin, request, queryset, "xli") - - -@admin.action(description="Import candidate URLs from LRM Dev Server") -def import_candidate_urls_lrm_dev_server(modeladmin, request, queryset): - import_candidate_urls_from_api_caller(modeladmin, request, queryset, "lrm_dev") - - -@admin.action(description="Import candidate URLs from LRM QA Server") -def import_candidate_urls_lrm_qa_server(modeladmin, request, queryset): - import_candidate_urls_from_api_caller(modeladmin, request, queryset, "lrm_qa") - - class ExportCsvMixin: def export_as_csv(self, request, queryset): meta = self.model._meta @@ -191,16 +69,8 @@ def export_as_csv(self, request, queryset): export_as_csv.short_description = "Export selected as csv" -class UpdateConfigMixin: - def update_config(self, request, queryset): - for collection in queryset: - collection.update_existing_config() - - update_config.short_description = "Update configs of selected" - - @admin.register(Collection) -class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin): +class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin): """Admin View for Collection""" fieldsets = ( @@ -287,12 +157,8 @@ def included_curated_urls_count(self, obj) -> int: ) search_fields = ("name", "url", "config_folder") actions = [ - generate_deployment_message, "export_as_csv", - "update_config", download_candidate_urls_as_csv, - fetch_full_text_lrm_dev_action, - fetch_full_text_xli_action, ] ordering = ("cleaning_order",) @@ -501,3 +367,56 @@ class CuratedUrlAdmin(TDAMMAdminMixin, admin.ModelAdmin): admin.site.register(DumpUrl, DumpUrlAdmin) admin.site.register(DeltaUrl, DeltaUrlAdmin) admin.site.register(CuratedUrl, CuratedUrlAdmin) + + +@admin.register(ScraperConfigOverride) +class ScraperConfigOverrideAdmin(admin.ModelAdmin): + """Curator-editable crawl overrides (WORKFLOW.md step 6).""" + + list_display = ( + "collection", + "max_pages", + "depth_limit", + "delay", + "concurrent_requests", + "obey_robots", + "include_subdomains", + ) + search_fields = ("collection__name", "collection__config_folder") + raw_id_fields = ("collection",) + + +@admin.register(ScrapeDispatch) +class ScrapeDispatchAdmin(admin.ModelAdmin): + """Read-only dispatch log for debugging scrape runs.""" + + list_display = ("collection", "dispatched_at", "ssm_command_id") + search_fields = ("collection__name", "collection__config_folder", "ssm_command_id") + list_filter = ("dispatched_at",) + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + +@admin.register(IndexDispatch) +class IndexDispatchAdmin(admin.ModelAdmin): + """Read-only dispatch log for debugging WEB_COSMOS index runs.""" + + list_display = ("collection", "target", "run_id", "dispatched_at", "completed_at") + search_fields = ("collection__name", "collection__config_folder", "run_id", "task_arn") + list_filter = ("target", "dispatched_at") + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False diff --git a/sde_collections/apps.py b/sde_collections/apps.py index f711fe47..8b2fad4b 100644 --- a/sde_collections/apps.py +++ b/sde_collections/apps.py @@ -4,3 +4,6 @@ class SdeCollectionsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "sde_collections" + + def ready(self): + from . import signals # noqa: F401 diff --git a/config_generation/__init__.py b/sde_collections/indexing/__init__.py similarity index 100% rename from config_generation/__init__.py rename to sde_collections/indexing/__init__.py diff --git a/sde_collections/indexing/dispatch.py b/sde_collections/indexing/dispatch.py new file mode 100644 index 00000000..5ea6b166 --- /dev/null +++ b/sde_collections/indexing/dispatch.py @@ -0,0 +1,92 @@ +"""Dispatch a WEB_COSMOS indexing run via cross-account ecs:RunTask. + +COSMOS assumes CosmosIndexingDispatchRole-{env} (or, with no role configured, uses its +own pipeline credentials — local dev) and starts one Fargate task with a command +override (not container-env overrides). Target->endpoint resolution is +tier-capped on the indexer side, so a dev dispatch can never reach prod AOSS. +""" + +import boto3 +from django.conf import settings + +from ..utils.aws import get_boto3_session + + +def _ecs_client(collection, target: str): + """ECS client under the dispatch role when INDEXING_DISPATCH_ROLE_ARN is set (the + deployed path: the instance role is the only principal the role trusts). With it + blank, the pipeline session's own credentials are used directly — local dev with + SDE_AWS_* keys that already carry ecs:RunTask + iam:PassRole.""" + session = get_boto3_session() + if not settings.INDEXING_DISPATCH_ROLE_ARN: + return session.client("ecs") + + session_name = f"cosmos-index-{target}-{collection.config_folder}"[:64] + credentials = session.client("sts").assume_role( + RoleArn=settings.INDEXING_DISPATCH_ROLE_ARN, + RoleSessionName=session_name, + )["Credentials"] + return boto3.client( + "ecs", + region_name=settings.AWS_REGION, + aws_access_key_id=credentials["AccessKeyId"], + aws_secret_access_key=credentials["SecretAccessKey"], + aws_session_token=credentials["SessionToken"], + ) + + +def run_index_task(collection, target: str, run_id: str) -> str: + """Assume the dispatch role (if configured) and RunTask; returns the task ARN.""" + for name in ("INDEXING_ECS_CLUSTER", "INDEXING_TASK_FAMILY"): + if not getattr(settings, name): + raise ValueError(f"{name} is not configured — cannot dispatch an index run") + + ecs = _ecs_client(collection, target) + + kwargs = { + "cluster": settings.INDEXING_ECS_CLUSTER, + "taskDefinition": settings.INDEXING_TASK_FAMILY, + "launchType": "FARGATE", + "count": 1, + "overrides": { + "containerOverrides": [ + { + "name": settings.INDEXING_CONTAINER_NAME, + # An ECS command override replaces the task definition's command + # wholesale, and the indexer image has no ENTRYPOINT — so the + # executable must be restated here, not just the flags. + "command": [ + "python3", + "api_scraper.py", + "--source", + "WEB_COSMOS", + "--collection", + collection.config_folder, + "--target", + target, + "--run-id", + run_id, + ], + } + ] + }, + } + subnets = [s for s in settings.INDEXING_SUBNETS.split(",") if s] + security_groups = [s for s in settings.INDEXING_SECURITY_GROUPS.split(",") if s] + if subnets or security_groups: + if not (subnets and security_groups): + raise ValueError("INDEXING_SUBNETS and INDEXING_SECURITY_GROUPS must both be set (or both blank)") + kwargs["networkConfiguration"] = { + "awsvpcConfiguration": { + "subnets": subnets, + "securityGroups": security_groups, + # Public subnets (dev's default VPC) need a public IP to reach S3/AOSS; + # a private subnet with NAT should set INDEXING_ASSIGN_PUBLIC_IP=False. + "assignPublicIp": "ENABLED" if settings.INDEXING_ASSIGN_PUBLIC_IP else "DISABLED", + } + } + + response = ecs.run_task(**kwargs) + if response.get("failures"): + raise RuntimeError(f"ecs:RunTask reported failures: {response['failures']}") + return response["tasks"][0]["taskArn"] diff --git a/sde_collections/indexing/export.py b/sde_collections/indexing/export.py new file mode 100644 index 00000000..8a0de69e --- /dev/null +++ b/sde_collections/indexing/export.py @@ -0,0 +1,106 @@ +"""Export a collection's curated set to the indexing hand-off bucket. + +Layout (fixed by the built WEB_COSMOS indexer — do not re-negotiate silently): + + s3://{SDE_INDEX_BUCKET}/curated_collections/{config_folder}/{run_id}/ + documents.jsonl + manifest.json <- written LAST = "export complete" + +COSMOS exports raw curated fields only. The indexer mints id/version itself +(web/web_processor.py is the sole owner of identity), verifies the JSONL line count +against manifest.document_count (skipping deletions on mismatch), and drops fields not +in its allow-list (tdamm_tag is exported but not indexed). +""" + +import json +import tempfile + +from django.conf import settings +from django.utils import timezone + +from ..models.collection_choice_fields import Divisions, DocumentTypes +from ..utils.aws import get_boto3_session + +SCHEMA_VERSION = 1 + + +def export_prefix(config_folder: str, run_id: str) -> str: + return f"curated_collections/{config_folder}/{run_id}" + + +def _label(choices_cls, value): + """Resolve a nullable choices int to its label; None stays None.""" + if value is None: + return None + return choices_cls(value).label + + +def _document_line(curated_url, collection_division, collection_document_type) -> dict: + """One JSONL line. Per-URL division/document_type only when they differ from the + collection default (the indexer broadcasts manifest values for missing fields).""" + line = { + "url": curated_url.url, + # Resolved at export time: manual/generated title wins over the scraped one. + "title": curated_url.generated_title or curated_url.scraped_title, + "full_text": curated_url.scraped_text, + "is_metadata_viewer": False, + } + + if curated_url.document_type is not None and curated_url.document_type != collection_document_type: + line["document_type"] = _label(DocumentTypes, curated_url.document_type) + if curated_url.division is not None and curated_url.division != collection_division: + line["division"] = _label(Divisions, curated_url.division) + + # tdamm_tag is a PairedFieldDescriptor (manual over ML), not a column — instance + # access only. Exported but not indexed (excluded from the indexer's version hash). + tdamm = curated_url.tdamm_tag + if tdamm: + line["tdamm_tag"] = list(tdamm) + + return line + + +def export_curated_to_s3(collection, target: str, run_id: str) -> int: + """Stream the curated set to S3; manifest last. Returns the exact document count.""" + from ..models.delta_url import CuratedUrl + + s3 = get_boto3_session().client("s3") + bucket = settings.SDE_INDEX_BUCKET + if not bucket: + raise ValueError("SDE_INDEX_BUCKET is not configured — cannot export") + prefix = export_prefix(collection.config_folder, run_id) + + # excluded is a queryset annotation, not a field — filtering it out here is what + # keeps curator exclusions from being published. + curated = CuratedUrl.objects.filter(collection=collection).exclude(excluded=True).iterator() + + count = 0 + # Spooled to disk: a crawled collection can be far larger than task memory. + with tempfile.TemporaryFile() as body: + for curated_url in curated: + line = _document_line(curated_url, collection.division, collection.document_type) + body.write(json.dumps(line, ensure_ascii=False).encode("utf-8") + b"\n") + count += 1 + body.seek(0) + s3.upload_fileobj(body, bucket, f"{prefix}/documents.jsonl") + + manifest = { + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "collection_key": collection.config_folder, + "collection_name": collection.name, + "division": _label(Divisions, collection.division), + "document_type": _label(DocumentTypes, collection.document_type), + "target": target, + "document_count": count, # the indexer verifies line count against this — must be exact + "exported_at": timezone.now().isoformat(), + "cosmos_workflow_status": collection.workflow_status, + } + # Written LAST: its presence is the "export complete" signal the indexer trusts. + s3.put_object( + Bucket=bucket, + Key=f"{prefix}/manifest.json", + Body=json.dumps(manifest).encode("utf-8"), + ContentType="application/json", + ) + return count diff --git a/sde_collections/indexing/run_status.py b/sde_collections/indexing/run_status.py new file mode 100644 index 00000000..0f350ba5 --- /dev/null +++ b/sde_collections/indexing/run_status.py @@ -0,0 +1,41 @@ +"""Read a WEB_COSMOS run's outcome from S3 — never ecs.describe_tasks. + +The indexer writes index_runs/{config_folder}/{run_id}/status.json last and +unconditionally, including on failure; on test runs it also writes validation.json +(the WORKFLOW.md steps 22–25 count/title QC report, produced indexer-side because +only it holds AOSS credentials). +""" + +import json + +from botocore.exceptions import ClientError +from django.conf import settings + +from ..utils.aws import get_boto3_session + +_MISSING_KEY_CODES = {"NoSuchKey", "404"} + + +def run_prefix(config_folder: str, run_id: str) -> str: + return f"index_runs/{config_folder}/{run_id}" + + +def _fetch_json(key: str): + s3 = get_boto3_session().client("s3") + try: + obj = s3.get_object(Bucket=settings.SDE_INDEX_BUCKET, Key=key) + except ClientError as e: + if e.response.get("Error", {}).get("Code") in _MISSING_KEY_CODES: + return None + raise + return json.loads(obj["Body"].read()) + + +def fetch_run_status(config_folder: str, run_id: str): + """status.json as a dict, or None while the run is still in flight.""" + return _fetch_json(f"{run_prefix(config_folder, run_id)}/status.json") + + +def fetch_validation_report(config_folder: str, run_id: str): + """validation.json (test runs only), or None if absent.""" + return _fetch_json(f"{run_prefix(config_folder, run_id)}/validation.json") diff --git a/sde_collections/management/commands/dispatch_scrape.py b/sde_collections/management/commands/dispatch_scrape.py new file mode 100644 index 00000000..b7fc4527 --- /dev/null +++ b/sde_collections/management/commands/dispatch_scrape.py @@ -0,0 +1,27 @@ +from django.core.management.base import BaseCommand, CommandError + +from sde_collections.models.collection import Collection +from sde_collections.tasks import dispatch_scrape_job + + +class Command(BaseCommand): + help = "Manually (re-)dispatch a scrape job for a collection to the crawl4ai crawler via SSM." + + def add_arguments(self, parser): + parser.add_argument("--collection", required=True, help="config_folder of the collection") + + def handle(self, *args, **options): + config_folder = options["collection"] + try: + collection = Collection.objects.get(config_folder=config_folder) + except Collection.DoesNotExist: + raise CommandError(f"No collection with config_folder={config_folder!r}") + + # Run synchronously so the operator sees the outcome immediately. + command_id = dispatch_scrape_job(collection.id) + if command_id is None: + raise CommandError( + f"Dispatch failed for {config_folder} — see task output above; " + f"collection is now marked Scraping Failed" + ) + self.stdout.write(self.style.SUCCESS(f"Dispatched {config_folder}: SSM command {command_id}")) diff --git a/sde_collections/management/commands/generate_configs.py b/sde_collections/management/commands/generate_configs.py deleted file mode 100644 index e77e4374..00000000 --- a/sde_collections/management/commands/generate_configs.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.core.management.base import BaseCommand - -from sde_collections.models.collection import Collection - - -class Command(BaseCommand): - help = "Export config for collections" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def handle(self, *args, **options) -> None: - collections = [ - "GISS Datasets and Derived Materials", - "GISS Publication List", - "NASA Global Climate Change", - "Goddard Institute for Space Studies", - "Earth Observer Publications", - "Our Changing Planet: The View from Space Images", - "NASA Sea Level Change", - "NASA Carbon Monitoring System", - "Algorithm Theoretical Basis Documents", - ] - - for collection_name in collections: - collection = Collection.objects.get(name=collection_name) - collection.export_config() - message = f"Successfully exported config for {collection.config_folder}" - self.stdout.write(self.style.SUCCESS(message)) diff --git a/sde_collections/management/commands/import_from_sinequa.py b/sde_collections/management/commands/import_from_sinequa.py deleted file mode 100644 index fa42066a..00000000 --- a/sde_collections/management/commands/import_from_sinequa.py +++ /dev/null @@ -1,19 +0,0 @@ -from django.core.management.base import BaseCommand - -from sde_collections.models.collection import Collection - - -class Command(BaseCommand): - help = "Load scraped URLs into the database" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def handle(self, *args, **options) -> None: - for collection in Collection.objects.all(): - if collection.import_metadata_from_sinequa_config(): - message = f"Successfully imported metadata from {collection.config_folder}" - self.stdout.write(self.style.SUCCESS(message)) - else: - message = f"Failed to import metadata from {collection.name}" - self.stdout.write(self.style.ERROR(message)) diff --git a/sde_collections/management/commands/ingest_scrape_results.py b/sde_collections/management/commands/ingest_scrape_results.py new file mode 100644 index 00000000..c7fa9e8c --- /dev/null +++ b/sde_collections/management/commands/ingest_scrape_results.py @@ -0,0 +1,28 @@ +from django.core.management.base import BaseCommand, CommandError + +from sde_collections.models.collection import Collection +from sde_collections.tasks import ingest_scraped_collection + + +class Command(BaseCommand): + help = ( + "Manually ingest completed crawl results from S3 for a collection. Skips the " + "status compare-and-swap (explicit operator intent) and ignores result freshness, " + "but still deletes existing DumpUrls first, so re-runs are idempotent." + ) + + def add_arguments(self, parser): + parser.add_argument("--collection", required=True, help="config_folder of the collection") + + def handle(self, *args, **options): + config_folder = options["collection"] + try: + collection = Collection.objects.get(config_folder=config_folder) + except Collection.DoesNotExist: + raise CommandError(f"No collection with config_folder={config_folder!r}") + + # Run synchronously so the operator sees the outcome immediately. + result = ingest_scraped_collection(collection.id, claim=False) + if result is None: + raise CommandError(f"Ingest failed for {config_folder} — see task output above") + self.stdout.write(self.style.SUCCESS(str(result))) diff --git a/sde_collections/management/commands/load_urls_from_api.py b/sde_collections/management/commands/load_urls_from_api.py deleted file mode 100644 index fb45b0f7..00000000 --- a/sde_collections/management/commands/load_urls_from_api.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.core.management.base import BaseCommand - -from sde_collections.tasks import import_candidate_urls_from_api - - -class Command(BaseCommand): - help = "Load scraped URLs into the database" - - def handle(self, *args, **options): - import_candidate_urls_from_api() - - self.stdout.write(self.style.SUCCESS("Successfully loaded urls from the test server")) diff --git a/sde_collections/management/commands/push_to_github.py b/sde_collections/management/commands/push_to_github.py deleted file mode 100644 index 2b47df6e..00000000 --- a/sde_collections/management/commands/push_to_github.py +++ /dev/null @@ -1,34 +0,0 @@ -from django.core.management.base import BaseCommand - -from sde_collections.models.collection import Collection -from sde_collections.models.collection_choice_fields import WorkflowStatusChoices -from sde_collections.utils.github_helper import GitHubHandler - - -class Command(BaseCommand): - help = "Push config to github. Takes comma-separated config_folder list as argument." - - def add_arguments(self, parser): - parser.add_argument("config_folders", nargs="*", type=str, default=[]) - - @staticmethod - def _get_names(collections): - return list(collections.values_list("name", flat=True)) - - def handle(self, *args, **options): - selected_collections = Collection.objects.filter(config_folders=options["config_folders"]) - curated_collections = selected_collections.filter(workflow_status=WorkflowStatusChoices.CURATED) - uncurated_collections = selected_collections.exclude(workflow_status=WorkflowStatusChoices.CURATED) - - gh = GitHubHandler(curated_collections) - gh.push_to_github() - - self.stdout.write(self.style.SUCCESS("Successfully pushed: %s" % self._get_names(curated_collections))) - - if uncurated_collections: - self.stdout.write( - self.style.ERROR( - "The following collections could not be pushed because the workflow status was not Curated %s" - % self._get_names(uncurated_collections) - ) - ) diff --git a/sde_collections/management/commands/sync_all_with_github.py b/sde_collections/management/commands/sync_all_with_github.py deleted file mode 100644 index aed91e6d..00000000 --- a/sde_collections/management/commands/sync_all_with_github.py +++ /dev/null @@ -1,41 +0,0 @@ -import json - -from django.core.management.base import BaseCommand - -from sde_collections.models.collection import Collection -from sde_collections.models.collection_choice_fields import Divisions, SourceChoices -from sde_collections.utils.github_helper import GitHubHandler - - -class Command(BaseCommand): - help = ( - "Sync all collections with GitHub. Takes comma-separated config_folder list as argument." - "If no argument is provided, all collections will be synced" - ) - - def add_arguments(self, parser): - parser.add_argument("config_folders", nargs="*", type=str, default=[]) - - # @staticmethod - # def _get_names(collections): - # return list(collections.values_list("name", flat=True)) - - def handle(self, *args, **options): - gh = GitHubHandler(collections=Collection.objects.none()) - collections = gh.get_collections_from_github() - - with open("github_collections.json", "w") as f: - json.dump(collections, f) - - for collection in collections: - Collection.objects.create( - config_folder=collection["config_folder"], - name=collection["name"], - url=collection["url"], - division=Divisions.lookup_by_text(collection["division"]), - document_type=collection["document_type"], - source=SourceChoices.BOTH, - connector=collection["connector"], - ) - - self.stdout.write(self.style.SUCCESS("Successfully synced")) diff --git a/sde_collections/migrations/0078_alter_collection_workflow_status_and_more.py b/sde_collections/migrations/0078_alter_collection_workflow_status_and_more.py new file mode 100644 index 00000000..feb86005 --- /dev/null +++ b/sde_collections/migrations/0078_alter_collection_workflow_status_and_more.py @@ -0,0 +1,143 @@ +# Generated by Django 4.2.9 on 2026-08-13 19:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0077_alter_candidateurl_tdamm_tag_manual_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="collection", + name="workflow_status", + field=models.IntegerField( + choices=[ + (1, "Research in Progress"), + (2, "Ready for Engineering"), + (3, "Engineering in Progress"), + (4, "Ready for Curation"), + (5, "Curation in Progress"), + (6, "Curated"), + (7, "Quality Fixed"), + (8, "Secret Deployment Started"), + (9, "Secret Deployment Failed"), + (10, "Ready for LRM Quality Check"), + (11, "Ready for Quality Check"), + (12, "QC: Failed"), + (18, "QC: Minor Issues"), + (13, "QC: Perfect"), + (14, "Prod: Perfect"), + (15, "Prod: Minor Issues"), + (16, "Prod: Major Issues"), + (17, "Code Merge Pending"), + (19, "Delete from Prod"), + (20, "Indexing Finished on LRM Dev"), + (21, "Scraping Successful"), + (22, "Test Indexing"), + (23, "Scraping Failed"), + (24, "Indexing Failed on Test"), + (25, "Indexing Failed on Prod"), + (26, "Production Indexing"), + ], + default=1, + ), + ), + migrations.AlterField( + model_name="deltadivisionpattern", + name="match_pattern_type", + field=models.IntegerField(choices=[(1, "Individual URL Pattern"), (2, "Multi-URL Pattern")], default=2), + ), + migrations.AlterField( + model_name="deltadocumenttypepattern", + name="match_pattern_type", + field=models.IntegerField(choices=[(1, "Individual URL Pattern"), (2, "Multi-URL Pattern")], default=2), + ), + migrations.AlterField( + model_name="deltaexcludepattern", + name="match_pattern_type", + field=models.IntegerField(choices=[(1, "Individual URL Pattern"), (2, "Multi-URL Pattern")], default=2), + ), + migrations.AlterField( + model_name="deltaincludepattern", + name="match_pattern_type", + field=models.IntegerField(choices=[(1, "Individual URL Pattern"), (2, "Multi-URL Pattern")], default=2), + ), + migrations.AlterField( + model_name="deltatitlepattern", + name="match_pattern_type", + field=models.IntegerField(choices=[(1, "Individual URL Pattern"), (2, "Multi-URL Pattern")], default=2), + ), + migrations.AlterField( + model_name="workflowhistory", + name="old_status", + field=models.IntegerField( + choices=[ + (1, "Research in Progress"), + (2, "Ready for Engineering"), + (3, "Engineering in Progress"), + (4, "Ready for Curation"), + (5, "Curation in Progress"), + (6, "Curated"), + (7, "Quality Fixed"), + (8, "Secret Deployment Started"), + (9, "Secret Deployment Failed"), + (10, "Ready for LRM Quality Check"), + (11, "Ready for Quality Check"), + (12, "QC: Failed"), + (18, "QC: Minor Issues"), + (13, "QC: Perfect"), + (14, "Prod: Perfect"), + (15, "Prod: Minor Issues"), + (16, "Prod: Major Issues"), + (17, "Code Merge Pending"), + (19, "Delete from Prod"), + (20, "Indexing Finished on LRM Dev"), + (21, "Scraping Successful"), + (22, "Test Indexing"), + (23, "Scraping Failed"), + (24, "Indexing Failed on Test"), + (25, "Indexing Failed on Prod"), + (26, "Production Indexing"), + ], + null=True, + ), + ), + migrations.AlterField( + model_name="workflowhistory", + name="workflow_status", + field=models.IntegerField( + choices=[ + (1, "Research in Progress"), + (2, "Ready for Engineering"), + (3, "Engineering in Progress"), + (4, "Ready for Curation"), + (5, "Curation in Progress"), + (6, "Curated"), + (7, "Quality Fixed"), + (8, "Secret Deployment Started"), + (9, "Secret Deployment Failed"), + (10, "Ready for LRM Quality Check"), + (11, "Ready for Quality Check"), + (12, "QC: Failed"), + (18, "QC: Minor Issues"), + (13, "QC: Perfect"), + (14, "Prod: Perfect"), + (15, "Prod: Minor Issues"), + (16, "Prod: Major Issues"), + (17, "Code Merge Pending"), + (19, "Delete from Prod"), + (20, "Indexing Finished on LRM Dev"), + (21, "Scraping Successful"), + (22, "Test Indexing"), + (23, "Scraping Failed"), + (24, "Indexing Failed on Test"), + (25, "Indexing Failed on Prod"), + (26, "Production Indexing"), + ], + default=1, + ), + ), + ] diff --git a/sde_collections/migrations/0079_scraperconfigoverride_scrapedispatch.py b/sde_collections/migrations/0079_scraperconfigoverride_scrapedispatch.py new file mode 100644 index 00000000..29e42df9 --- /dev/null +++ b/sde_collections/migrations/0079_scraperconfigoverride_scrapedispatch.py @@ -0,0 +1,62 @@ +# Generated by Django 4.2.9 on 2026-08-13 20:03 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0078_alter_collection_workflow_status_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ScraperConfigOverride", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("max_pages", models.PositiveIntegerField(blank=True, help_text="Crawler cap: 100,000", null=True)), + ("depth_limit", models.PositiveIntegerField(blank=True, null=True)), + ( + "delay", + models.FloatField( + blank=True, help_text="Seconds between requests (crawler default 0.25)", null=True + ), + ), + ("concurrent_requests", models.PositiveSmallIntegerField(blank=True, null=True)), + ("obey_robots", models.BooleanField(blank=True, null=True)), + ("include_subdomains", models.BooleanField(blank=True, null=True)), + ( + "collection", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="scraper_config", + to="sde_collections.collection", + ), + ), + ], + options={ + "verbose_name": "Scraper Config Override", + }, + ), + migrations.CreateModel( + name="ScrapeDispatch", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("dispatched_at", models.DateTimeField(auto_now_add=True)), + ("ssm_command_id", models.CharField(max_length=64)), + ( + "collection", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="scrape_dispatches", + to="sde_collections.collection", + ), + ), + ], + options={ + "verbose_name_plural": "Scrape Dispatches", + "ordering": ["-dispatched_at"], + }, + ), + ] diff --git a/sde_collections/migrations/0080_indexdispatch.py b/sde_collections/migrations/0080_indexdispatch.py new file mode 100644 index 00000000..9190377e --- /dev/null +++ b/sde_collections/migrations/0080_indexdispatch.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.9 on 2026-08-13 21:27 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0079_scraperconfigoverride_scrapedispatch"), + ] + + operations = [ + migrations.CreateModel( + name="IndexDispatch", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("run_id", models.CharField(max_length=64)), + ("target", models.CharField(choices=[("test", "test"), ("prod", "prod")], max_length=8)), + ("task_arn", models.CharField(blank=True, default="", max_length=256)), + ("previous_workflow_status", models.IntegerField(blank=True, null=True)), + ("dispatched_at", models.DateTimeField(auto_now_add=True)), + ("completed_at", models.DateTimeField(blank=True, null=True)), + ( + "collection", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="index_dispatches", + to="sde_collections.collection", + ), + ), + ], + options={ + "verbose_name_plural": "Index Dispatches", + "ordering": ["-dispatched_at"], + }, + ), + ] diff --git a/sde_collections/models/README.md b/sde_collections/models/README.md index 3c326f6a..97502fc4 100644 --- a/sde_collections/models/README.md +++ b/sde_collections/models/README.md @@ -13,3 +13,9 @@ A system for managing collections of URLs through pattern-based rules and status - [Pattern Unapplication Logic](./README_UNAPPLY_LOGIC.md) - [Collection Status Workflows](./README_STATUS_TRIGGERS.md) - Collection progression and automated triggers - [Reindexing Status System](./README_REINDEXING_STATUSES.md) - Status management for reindexing collections + +## Scraping and Indexing Models + +- [`ScraperConfigOverride`](./scraper_config.py) - Per-collection overrides (`max_pages`, `depth_limit`, `delay`, `concurrent_requests`, `obey_robots`, `include_subdomains`) merged onto the crawler's own defaults. All fields are nullable; only non-null values reach the job JSON. +- [`ScrapeDispatch`](./scraper_config.py) - One row per SSM scrape dispatch, recording `ssm_command_id` and `dispatched_at`. `dispatched_at` is the poller's freshness reference (older S3 results belong to a previous run) and the stall-timeout start time. +- [`IndexDispatch`](./indexing.py) - One row per indexing run, recording `collection`, `run_id`, `target` (test or prod), `task_arn`, `previous_workflow_status`, and `dispatched_at`, plus `completed_at` once the poller resolves the run. diff --git a/sde_collections/models/README_LIFECYCLE.md b/sde_collections/models/README_LIFECYCLE.md index 8b9a8cd2..fcbb26d2 100644 --- a/sde_collections/models/README_LIFECYCLE.md +++ b/sde_collections/models/README_LIFECYCLE.md @@ -32,29 +32,36 @@ The classification process analyzes content to automatically add metadata, inclu - TDAMM tags for Astrophysics content - Division classification for General content +Classification is optional and is gated on the `INFERENCE_ENABLED` setting, which +defaults to `False`. The inference pipeline is currently dormant, so in the default +configuration no classification stage runs at all. + ### When Classification Happens -Classification occurs after DumpUrls are created but before they are migrated to DeltaUrls: +Classification, when enabled, occurs after DumpUrls are created but before they are migrated to DeltaUrls: 1. DumpUrls are created from scraped content -2. Classification models analyze DumpUrl content -3. Classification results are applied to DumpUrls -4. DumpUrls (with enhanced metadata) are migrated to DeltaUrls +2. `Collection.queue_necessary_classifications()` is called +3. If `INFERENCE_ENABLED` is `False`, migration is queued immediately and steps 4–6 are skipped +4. Classification models analyze DumpUrl content +5. Classification results are applied to DumpUrls +6. DumpUrls (with enhanced metadata) are migrated to DeltaUrls ### Classification Types - **TDAMM Classification**: Applied to Astrophysics collections to tag content related to multi-messenger astronomy - **Division Classification**: Applied to General collections to suggest appropriate divisions ### Classification Flow -1. Check if collection needs classification based on division type -2. Queue appropriate classification jobs -3. Process classifications asynchronously -4. Apply classification results to DumpUrls -5. Initiate migration to DeltaUrls once all classifications complete +1. If `INFERENCE_ENABLED` is `False`, immediately queue `migrate_dump_to_delta_and_handle_status_transistions` and stop +2. Check if collection needs classification based on its configuration +3. Queue appropriate classification jobs; collections needing none go straight to migration +4. Process classifications asynchronously +5. Apply classification results to DumpUrls +6. Initiate migration to DeltaUrls once all classifications complete ## Pattern Application ### When Patterns Are Applied Patterns are applied in two scenarios: -1. During migration from Dump to Delta (after classifications are complete) +1. During migration from Dump to Delta (after classifications are complete, if any ran) 2. When a new pattern is created/updated Patterns are NOT applied during promotion. The effects of patterns (modified titles, document types, etc.) are carried through to CuratedUrls during promotion, but the patterns themselves don't reapply. @@ -69,7 +76,7 @@ Patterns are NOT applied during promotion. The effects of patterns (modified tit ### Overview Migration converts DumpUrls to DeltaUrls, preserving all fields and applying patterns. This process happens when: -- New content is scraped and classified +- New content is scraped (and classified, if `INFERENCE_ENABLED` is on) - Content is reindexed - Collection is being prepared for curation @@ -268,9 +275,10 @@ curated_url = CuratedUrl( - Pattern-set values take precedence over original values ### Classification Behavior +- Classifications only run when `INFERENCE_ENABLED` is on; with the flag off (the default) migration runs immediately after the DumpUrls are created - Classifications only run on DumpUrls before migration to DeltaUrls - Classification results become regular field values and persist through promotion -- Migration to DeltaUrls waits for all classifications to complete +- When classifications are queued, migration to DeltaUrls waits for all of them to complete ### Pattern Behavior - Patterns only apply during migration or when patterns themselves are created/updated diff --git a/sde_collections/models/README_MANUAL_TESTING.md b/sde_collections/models/README_MANUAL_TESTING.md index 2be75f50..f35d2a7b 100644 --- a/sde_collections/models/README_MANUAL_TESTING.md +++ b/sde_collections/models/README_MANUAL_TESTING.md @@ -1,166 +1,217 @@ # COSMOS Curation System Testing Guide -## Resources -There are 14 collections which have been reindexed on dev and can have their statuses changed to `REINDEXING_FINISHED` to test url importing. The collections and their counts can be seen [here](https://docs.google.com/spreadsheets/d/1z_YeTwsyadW6ywPsahUElnf8X65gP7t7UyaO7sVqGiI/edit?gid=1316450061#gid=1316450061). +Manual acceptance tests for the curation pipeline. This is the curator-facing counterpart to +[`LOCAL_VERIFICATION_GUIDE.md`](../../LOCAL_VERIFICATION_GUIDE.md), which verifies a local install +from a developer's point of view. Here the question is whether the workflow behaves correctly for +the people who use it. -## Test Flow 1: Basic URL Collection Lifecycle +The pipeline these tests exercise is described in [`WORKFLOW.md`](../../WORKFLOW.md), and what each +status change triggers is in [`README_STATUS_TRIGGERS.md`](./README_STATUS_TRIGGERS.md). + +## Before you start + +Every dispatch-gating setting defaults blank or off, so on an unwired host **nothing will happen** +when you change a status and the tests below will all appear to fail. Confirm first: + +- `CRAWLER_INSTANCE_ID` and `SDE_S3_BUCKET` are set, and `SCRAPE_POLL_ENABLED=true`, for anything + involving a scrape. +- `SDE_INDEX_BUCKET`, `INDEXING_ECS_CLUSTER`, `INDEXING_TASK_FAMILY`, `INDEXING_DISPATCH_ROLE_ARN`, + `INDEXING_SUBNETS`, `INDEXING_SECURITY_GROUPS` are set, and `INDEX_POLL_ENABLED=true`, for + anything involving indexing. +- `manage.py migrate` has been run **since** those flags were last changed. The poller schedules are + `django_celery_beat` rows written by a `post_migrate` receiver, so a restart alone does not enable + them. + +Pick a small collection. Avoid one whose documents are already in the target index under a +different id scheme — the indexer refuses those deliberately, which looks like a failure but is +correct behaviour. + +## Test Flow 1: Scrape dispatch and ingest + +### Objective + +Verify a collection can be scraped and its results ingested. + +### Test Cases + +#### 1.1 Dispatch + +1. Set a collection's workflow status to **Ready for Engineering**. +2. Confirm a `ScrapeDispatch` row is created, carrying an `ssm_command_id` and `dispatched_at`. +3. Confirm the job JSON lands in the crawler's inbox. It is written via a temporary file and then + moved, so the watcher should never see a partial file. +4. Failure path: with `CRAWLER_INSTANCE_ID` unset or the instance unreachable, confirm the status + moves to **Scraping Failed** rather than hanging. +5. Cap check: request a page count above the crawler's cap (100,000) and confirm the job is + rejected outright rather than silently clamped. + +#### 1.2 Ingest + +1. Wait for the crawler to write its results. The poller runs every 5 minutes. +2. Confirm the status moves to **Scraping Successful**, then on to **Ready for Curation**. +3. Confirm `DumpUrl` rows appear, then `DeltaUrl` rows after migration. +4. Confirm a summary is posted to Slack. +5. Staleness: confirm results older than the dispatch are ignored — the poller only accepts a + summary written *after* `ScrapeDispatch.dispatched_at`. +6. Empty result: a scrape returning zero pages should land on **Scraping Failed**, not + **Scraping Successful** with an empty collection. +7. Stall: with nothing fresh for longer than `SCRAPE_STALL_TIMEOUT_HOURS` (default 24), confirm the + collection lands on **Scraping Failed**. + +Expected results: + +- Each status transition fires exactly once; re-saving a collection does not re-dispatch. +- Claiming is atomic, so two pollers running concurrently cannot double-ingest. +- With `INFERENCE_ENABLED=False` (the default) classification is skipped entirely and migration to + DeltaUrls runs immediately. + +## Test Flow 2: Curation and the test-indexing hand-off ### Objective -Verify the complete lifecycle of a URL collection from initial creation through curation to production. -### Prerequisites -- Access to dev environment -- Test collection created -- Sample URLs ready for testing +Verify curation promotes correctly and hands off to the indexer. ### Test Cases -#### 1.1 Collection Status Progression -1. Create new collection in `RESEARCH_IN_PROGRESS` status -2. Verify initial scraper and indexer configs are created when moved to `READY_FOR_ENGINEERING` -3. Progress through `ENGINEERING_IN_PROGRESS` to `INDEXING_FINISHED_ON_DEV` -4. Confirm full text fetch triggers automatically -5. Verify status updates to `READY_FOR_CURATION` -6. Check plugin config creation -7. Move through `CURATION_IN_PROGRESS` to `CURATED` -8. Verify DeltaUrls promotion to CuratedUrls -9. Test quality check status changes (`QUALITY_CHECK_PERFECT/MINOR`) -10. Confirm collection appears in public query after PR merge - -#### 1.2 Data State Transitions -1. Verify DumpUrls are created during indexing -2. Test migration from DumpUrls to DeltaUrls -3. Confirm field preservation during transitions -4. Check promotion from DeltaUrls to CuratedUrls -5. Verify all metadata transfers correctly - -Expected Results: -- Each status transition triggers appropriate automated actions -- Data integrity maintained through all transitions -- Correct config generation at each stage -- Proper public visibility after final approval - -## Test Flow 2: Pattern System Functionality +#### 2.1 Promotion and dispatch + +1. Curate the collection — apply include/exclude patterns, title and document-type changes. +2. Set the status to **Curated**. +3. Confirm `DeltaUrl`s are promoted to `CuratedUrl`s and the Delta set is cleared. +4. Confirm an export appears in S3 under `curated_collections/{config_folder}/{run_id}/`, with + `documents.jsonl` written first and `manifest.json` **last**. The manifest's `document_count` + must exactly match the JSONL line count. +5. Confirm excluded URLs are absent from the export. +6. Confirm an `IndexDispatch` row is created with a `run_id`, `target=test`, and a `task_arn`, and + that the status moves to **Test Indexing**. +7. Empty-curation failure path: a collection with no curated URLs should fail dispatch and land on + **Indexing Failed on Test** rather than exporting an empty file. + +#### 2.2 Reading the result + +1. The poller runs every 2 minutes. When the indexer finishes, confirm the collection **stays** in + **Test Indexing** — a successful test run is not an automatic promotion. +2. Confirm the validation report is posted to Slack. +3. Failure path: confirm a failed run, an unrecognised result, or no result at all within + `INDEX_STALL_TIMEOUT_HOURS` (default 6) lands on **Indexing Failed on Test**. + +## Test Flow 3: Quality check and production indexing ### Objective -Test the creation, application, and interaction of different pattern types. -### Prerequisites -- Collection with sample URLs -- Mix of different URL types and structures +Verify the QC decision drives the production hand-off. ### Test Cases -#### 2.1 Include/Exclude Patterns -1. Create exclude pattern for specific directory +1. From **Test Indexing**, set **QC: Perfect** and confirm a second indexing run is dispatched with + a new `run_id`, and the status moves to **Production Indexing**. +2. On success, confirm the status lands on **Prod: Perfect**. +3. Repeat from **QC: Minor Issues** and confirm the terminal status is **Prod: Minor Issues** — the + distinction must survive the round trip. +4. Failure path: confirm a failed production run lands on **Indexing Failed on Prod**. + +## Test Flow 4: Re-scrape and re-curation + +### Objective + +Verify an already-published collection can be refreshed. + +### Test Cases + +1. Set `reindexing_status` to **Re-Indexing Needed** and confirm a *new* `ScrapeDispatch` row is + created. +2. Confirm the poller ignores the previous scrape's S3 output until fresh results land. +3. Confirm the collection reaches **Ready for Re-Curation**. +4. Confirm patterns are reapplied to the new URLs and that manual, per-URL changes are preserved. +5. Set **Re-Curation Finished** and confirm promotion runs. Note the causality: the curator sets + this status, and that triggers promotion — not the reverse. +6. Confirm **Re-Indexing Finished** triggers nothing on its own; the ingest task sets that status + itself, and a second trigger would double-fire. + +## Test Flow 5: Pattern system + +### Objective + +Test creation, application, and interaction of pattern types. This area is independent of the +scrape/index plumbing and is unchanged by the rewiring. + +### Test Cases + +#### 5.1 Include/exclude patterns + +1. Create an exclude pattern for a directory: ```python pattern = "https://example.com/internal/*" ``` -2. Create include pattern for specific file within excluded directory +2. Create an include pattern for one file inside it: ```python pattern = "https://example.com/internal/public-doc.html" ``` -3. Verify include pattern overrides exclude pattern -4. Test wildcard pattern matching -5. Check pattern precedence rules +3. Verify the include pattern overrides the exclude. +4. Test wildcard matching and precedence rules. +5. Confirm excluded URLs do not reach the export in Test Flow 2. + +#### 5.2 Modification patterns -#### 2.2 Modification Patterns 1. Create overlapping title patterns: ```python pattern1 = "*/docs/* → title='Documentation'" pattern2 = "*/docs/api/* → title='API Reference'" ``` -2. Create division patterns with different specificity -3. Test document type patterns with wildcards -4. Verify "smallest set priority" resolution -5. Check pattern application during migrations - -#### 2.3 Pattern Removal Scenarios -1. Test removing pattern affecting only Delta URLs -2. Remove pattern affecting Curated URLs -3. Verify handling of multiple pattern effects -4. Test manual change preservation -5. Check cleanup procedures - -Expected Results: -- Pattern precedence rules correctly applied -- Proper handling of overlapping patterns -- Manual changes preserved during pattern operations -- Correct reversal of pattern effects on removal - -## Test Flow 3: Reindexing Workflow - -### Objective -Verify the reindexing process and status management. - -### Prerequisites -- Existing collection in production -- Access to both dev and prod environments - -### Test Cases - -#### 3.1 Reindexing Status Progression -1. Change status from `REINDEXING_NOT_NEEDED` to `REINDEXING_NEEDED_ON_DEV` -2. Complete reindexing and update to `REINDEXING_FINISHED_ON_DEV` -3. Verify automatic full text fetch -4. Confirm status update to `REINDEXING_READY_FOR_CURATION` -5. Progress through `REINDEXING_CURATED` -6. Final update to `REINDEXING_INDEXED_ON_PROD` - -#### 3.2 Data Handling During Reindex -1. Verify existing DumpUrls are cleared -2. Check new full text data processing -3. Test DumpUrl to DeltaUrl migration -4. Verify pattern reapplication -5. Confirm CuratedUrl updates - -Expected Results: -- Proper status progression through reindexing -- Data integrity maintained -- Patterns correctly reapplied -- Existing customizations preserved - -## Edge Cases and Stress Testing - -### URL Pattern Edge Cases -1. Test URLs with/without trailing slashes -2. Verify handling of overlapping wildcards -3. Check pattern resolution with equal URL count matches -4. Test maximum pattern chain depth -5. Verify handling of malformed URLs - -### Status Transition Edge Cases -1. Test interrupted transitions -2. Verify handling of failed automated actions -3. Check concurrent status updates -4. Test invalid status progressions -5. Verify recovery procedures - -### Data Volume Testing -1. Test with large number of URLs (>100k) -2. Check pattern application performance -3. Verify migration speed with large datasets -4. Test memory usage during bulk operations -5. Check system response under heavy concurrent access - -## Common Issues to Watch For - -1. Pattern Precedence - - Multiple patterns affecting same URL - - Include/exclude pattern conflicts - - Resolution of equal-specificity patterns - -2. Data Integrity - - Field preservation during transitions - - Manual change retention - - Pattern effect tracking - -3. Performance - - Large collection handling - - Multiple pattern application - - Status transition timing - -4. Status Management - - Automated trigger reliability - - Status update race conditions - - Recovery from failed transitions +2. Create division patterns of differing specificity. +3. Test document-type patterns with wildcards. +4. Verify "smallest set priority" resolution. +5. Check pattern application during migrations. + +#### 5.3 Pattern removal + +1. Remove a pattern affecting only Delta URLs. +2. Remove one affecting Curated URLs. +3. Verify handling where several patterns affect the same URL. +4. Confirm manual changes are preserved. + +Expected results: + +- Precedence rules are applied consistently. +- Manual changes survive pattern operations. +- Removing a pattern reverses its effects. + +## Edge cases + +### URL patterns + +1. URLs with and without trailing slashes. +2. Overlapping wildcards. +3. Equal URL-count matches. +4. Maximum pattern chain depth. +5. Malformed URLs. + +### Status transitions + +1. Interrupted transitions. +2. Failed automated actions — every failure path should reach a terminal *failed* status, never + leave the collection in an in-progress state indefinitely. +3. Concurrent status updates. +4. Invalid progressions. +5. Recovery: confirm a collection that reached **Scraping Failed** or **Indexing Failed on Test** + can be re-driven through the workflow without manual database surgery. + +### Data volume + +1. Large collections (>100k URLs). +2. Pattern application performance. +3. Migration speed on large datasets. +4. Memory use during bulk operations. + +## Common issues to watch for + +1. **Nothing happens on a status change.** Almost always an unwired setting or a missing `migrate` + — check the prerequisites above before investigating anything else. +2. **Pattern precedence.** Multiple patterns on one URL, include/exclude conflicts, equal-specificity + resolution. +3. **Data integrity.** Field preservation across Dump → Delta → Curated, retention of manual + changes, pattern effect tracking. +4. **Stale results.** The scrape poller uses dispatch time to reject old output; index runs are + namespaced by `run_id`. A collection picking up a previous run's data is a bug worth reporting + in detail. +5. **Status races.** Two workers acting on one collection, or a status change firing twice. diff --git a/sde_collections/models/README_PATTERN_SYSTEM.md b/sde_collections/models/README_PATTERN_SYSTEM.md index b8381747..a159b149 100644 --- a/sde_collections/models/README_PATTERN_SYSTEM.md +++ b/sde_collections/models/README_PATTERN_SYSTEM.md @@ -8,7 +8,7 @@ The pattern system is designed to manage and track changes to URLs in a content ### URL States - **Curated URLs**: Production-ready, approved content - **Delta URLs**: Work-in-progress changes, additions, or deletions to curated content -- **Dump URLs**: Raw content from the dev server +- **Dump URLs**: Raw content ingested from the crawl4ai scrape results in S3 ### Pattern Types 1. **Exclude Patterns**: Mark URLs for exclusion from the collection diff --git a/sde_collections/models/README_REINDEXING_STATUSES.md b/sde_collections/models/README_REINDEXING_STATUSES.md index 144a83c2..d93de96b 100644 --- a/sde_collections/models/README_REINDEXING_STATUSES.md +++ b/sde_collections/models/README_REINDEXING_STATUSES.md @@ -5,8 +5,8 @@ The typical reindexing status flow is: 1. `REINDEXING_NOT_NEEDED` ("Re-Indexing Not Needed") → Default state -2. `REINDEXING_NEEDED_ON_DEV` ("Re-Indexing Needed") → When reindexing is required -3. `REINDEXING_FINISHED_ON_DEV` ("Re-Indexing Finished") → After reindexing completes +2. `REINDEXING_NEEDED_ON_DEV` ("Re-Indexing Needed") → When a re-scrape is required +3. `REINDEXING_FINISHED_ON_DEV` ("Re-Indexing Finished") → After the scrape results are ingested 4. `REINDEXING_READY_FOR_CURATION` ("Ready for Re-Curation") → After dump URLs are migrated 5. `REINDEXING_CURATION_IN_PROGRESS` ("Re-Curation in Progress") → During active re-curation 6. `REINDEXING_CURATED` ("Re-Curation Finished") → After re-curation is complete @@ -18,21 +18,26 @@ The typical reindexing status flow is: - Default status for new collections - Applied to collections in early workflow stages (research, engineering, etc.) -### Reindexing Needed on LRM Dev +### Reindexing Needed - Variable name: `REINDEXING_NEEDED_ON_DEV` (2) -- Indicates collections that need to be reindexed on LRM Dev environment +- Indicates collections that need to be re-scraped - For collections that have already been indexed on production +- Manually set by a curator or engineer; triggers `dispatch_scrape_job`, which sends the + job to the crawl4ai host via SSM (the same dispatch/poll path as the initial workflow) -### Reindexing Finished on LRM Dev +### Reindexing Finished - Variable name: `REINDEXING_FINISHED_ON_DEV` (3) -- For collections that have completed reindexing on LRM Dev -- Currently managed manually by LRM team via admin interface +- For collections whose re-scrape has completed and whose results have been claimed +- Set automatically by `ingest_scraped_collection` as its atomic compare-and-swap claim + on the re-scrape path (`REINDEXING_NEEDED_ON_DEV` → `REINDEXING_FINISHED_ON_DEV`) +- This status deliberately triggers nothing: the ingest sets it, so a trigger here would + double-fire ### Ready for Re-Curation - Variable name: `REINDEXING_READY_FOR_CURATION` (4) -- Automatically set when: - - A collection's dump URLs are migrated to delta URLs AND there are curated URLs present - - Triggered by Collection.migrate_dump_to_delta() method +- Automatically set after a collection's dump URLs are migrated to delta URLs +- Set by the `migrate_dump_to_delta_and_handle_status_transistions` task, for collections + that entered migration at `REINDEXING_FINISHED_ON_DEV` ### Re-Curation in Progress - Variable name: `REINDEXING_CURATION_IN_PROGRESS` (5) @@ -42,33 +47,42 @@ The typical reindexing status flow is: ### Re-Curation Finished - Variable name: `REINDEXING_CURATED` (6) -- Automatically set when: - - Delta URLs are promoted to curated URLs AND there are curated URLs present - - Triggered by Collection.promote_to_curated() method +- Manually set by the curator when re-curation is finished +- Setting it is what *triggers* promotion: the `handle_workflow_status_change` receiver + calls `Collection.promote_to_curated()`, moving delta URLs to curated URLs ### Re-Indexed on Prod - Variable name: `REINDEXING_INDEXED_ON_PROD` (7) -- Currently managed manually via command line -- Future: Will be set automatically via plugin ping +- Manually set by a dev after the collection has been indexed on prod ### Key Code Locations for Automatic Changes -1. In migrate_dump_to_delta(): +1. In `ingest_scraped_collection()` (`sde_collections/tasks.py`) — the claim on the + re-scrape path: ```python -# After migrating, check if we should update reindexing status -curated_urls_count = self.curated_urls.count() -if curated_urls_count > 0: - self.reindexing_status = ReindexingStatusChoices.REINDEXING_READY_FOR_CURATION - self.save() +claimed = Collection.objects.filter( + id=collection_id, + reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV, +).update(reindexing_status=ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV) ``` -2. In promote_to_curated(): +2. In `migrate_dump_to_delta_and_handle_status_transistions()` (`sde_collections/tasks.py`): ```python -# After promoting, check if we should update reindexing status -curated_urls_count = self.curated_urls.count() -if curated_urls_count > 0: - self.reindexing_status = ReindexingStatusChoices.REINDEXING_CURATED - self.save() +# Check reindexing status transition +if initial_reindexing_status == ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV: + collection.reindexing_status = ReindexingStatusChoices.REINDEXING_READY_FOR_CURATION + collection.save() ``` -Note: All status changes are logged in the ReindexingHistory model for tracking purposes. +3. In `handle_workflow_status_change()` (`sde_collections/models/collection.py`) — the + status set by the curator drives the action, not the other way around: +```python +if instance.reindexing_status == ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV: + dispatch_scrape_job.delay(instance.id) +elif instance.reindexing_status == ReindexingStatusChoices.REINDEXING_CURATED: + instance.promote_to_curated() +``` + +Note: Status changes made through `Collection.save()` are logged in the ReindexingHistory +model for tracking purposes. The ingest claim above is a queryset `.update()`, which +bypasses the `post_save` receiver and so is not recorded there. diff --git a/sde_collections/models/README_STATUS_TRIGGERS.md b/sde_collections/models/README_STATUS_TRIGGERS.md index 8f8b8397..a5a21487 100644 --- a/sde_collections/models/README_STATUS_TRIGGERS.md +++ b/sde_collections/models/README_STATUS_TRIGGERS.md @@ -1,6 +1,9 @@ # Collection Status Workflows This document outlines the automated workflows triggered by status changes in Collections. +The single dispatcher is the `handle_workflow_status_change` post_save receiver in +`collection.py` (formerly `create_configs_on_status_change` — renamed in the Sinequa +retirement: it no longer creates configs). ## Workflow Status Transitions @@ -8,37 +11,46 @@ Collections progress through workflow statuses that trigger specific automated a ### Initial Flow 1. `RESEARCH_IN_PROGRESS` → `READY_FOR_ENGINEERING` - - Triggers: Creation of initial scraper and indexer configs - -2. `READY_FOR_ENGINEERING` → `ENGINEERING_IN_PROGRESS` → `INDEXING_FINISHED_ON_DEV` - - When indexing finishes, a developer changes the status to `INDEXING_FINISHED_ON_DEV` - - This will trigger a full text fetch from LRM dev - - If the fetch completes successfully, it updates the status to `READY_FOR_CURATION` - -3. `READY_FOR_CURATION` - - Triggers creation/update of plugin config - -4. `READY_FOR_CURATION` → `CURATION_IN_PROGRESS` → `CURATED` + - Triggers `dispatch_scrape_job`: a job JSON (seed + any `ScraperConfigOverride` + values) is written to the crawl4ai crawler's inbox on EC2 via SSM, and a + `ScrapeDispatch` row records the dispatch time + +2. Scrape completion (automated — no manual status change) + - The `poll_scrape_jobs` beat task (every 5 min, gated on `SCRAPE_POLL_ENABLED`) + watches S3 for a results summary fresher than the dispatch + - On completion, `ingest_scraped_collection` claims the collection + (→ `SCRAPING_SUCCESSFUL`), replaces its DumpUrls from S3, migrates to DeltaUrls, + and lands on `READY_FOR_CURATION` + - A zero-document crawl, a dispatch/ingest failure, or a stall past + `SCRAPE_STALL_TIMEOUT_HOURS` lands on `SCRAPING_FAILED` + +3. `READY_FOR_CURATION` → `CURATION_IN_PROGRESS` → `CURATED` - When curation finishes, the curator marks the collection as `CURATED` - - This triggers the promotion of DeltaUrls to CuratedUrls + - This promotes DeltaUrls to CuratedUrls **and** enqueues `index_collection_to_test` + (→ `TEST_INDEXING`) — the hand-off to the WEB_COSMOS indexing pipeline -5. Quality Check Flow: +4. Quality Check Flow: - During quality checks the curator can put the status as `QUALITY_CHECK_PERFECT/MINOR` - - These passing quality statuses will trigger the addition of the collection to the public query - - After the PR is merged and SDE Prod server is updated with the latest code, this collection will become visible + - These passing quality statuses enqueue `index_collection_to_prod` + (→ `PRODUCTION_INDEXING`) + - Indexing failures surface as `INDEXING_FAILED_ON_TEST` / `INDEXING_FAILED_ON_PROD` + +`INDEXING_FINISHED_ON_DEV` is a Sinequa-era status and no longer triggers anything. ### Reindexing Flow -After the main workflow, collections can enter a reindexing cycle: +After the main workflow, collections can enter a re-scrape cycle: 1. `REINDEXING_NOT_NEEDED` → `REINDEXING_NEEDED_ON_DEV` - - By default collections do not need reindexing - - They can be manually marked as reindexing needed on dev + - Manually marked; triggers `dispatch_scrape_job` (same dispatch/poll path as the + initial flow — this replaces the engineer manually re-running a Sinequa job) -2. `REINDEXING_NEEDED_ON_DEV` → `REINDEXING_FINISHED_ON_DEV` - - When re-indexing finishes, a developer changes the status to `REINDEXING_FINISHED_ON_DEV` - - This will trigger a full text fetch from LRM dev - - If the fetch completes successfully, it updates the status to `REINDEXING_READY_FOR_CURATION` +2. Scrape completion (automated) + - The poller watches the same S3 contract; ingest claims via + `REINDEXING_NEEDED_ON_DEV → REINDEXING_FINISHED_ON_DEV` and the migrate task + promotes to `REINDEXING_READY_FOR_CURATION` + - `REINDEXING_FINISHED_ON_DEV` itself deliberately triggers **nothing**: the ingest + sets it, so a trigger here would double-fire 3. `REINDEXING_READY_FOR_CURATION` → `REINDEXING_CURATED` - When re-curation finishes, the curator marks the collection as `REINDEXING_CURATED` @@ -47,18 +59,43 @@ After the main workflow, collections can enter a reindexing cycle: 4. `REINDEXING_CURATED` → `REINDEXING_INDEXED_ON_PROD` - After the collection has been indexed on Prod, a dev marks it as `REINDEXING_INDEXED_ON_PROD` -## Full Text Import Process +## Slack notifications + +Status-transition messages (`STATUS_CHANGE_NOTIFICATIONS`) are sent from the post_save +receiver — never from `Collection.save()` — so a message cannot be sent for a save that +then fails. The detailed ingest summary is posted by the migrate task after delta counts +exist. -The full text import process integrates with both workflows: +## Scrape Ingest Process -1. Clears existing DumpUrls for the collection -2. Fetches and processes new full text data in batches -3. Creates new DumpUrls +The S3 ingest (replacing the old Sinequa full-text import) integrates with both workflows: + +1. Claims the collection via an atomic status compare-and-swap (the transition is the lock) +2. Clears existing DumpUrls for the collection +3. Creates new DumpUrls from the scraped documents in S3 4. Migrates DumpUrls to DeltaUrls 5. Updates collection status based on context: - In main workflow: Updates to `READY_FOR_CURATION` - In reindexing: Updates to `REINDEXING_READY_FOR_CURATION` +## New Pipeline Statuses (crawl4ai scraper + web indexing) + +Statuses 21–26 support the Sinequa-replacement pipeline (see `WORKFLOW.md`). As of Phase 1 +they are selectable and rendered everywhere; their triggers land in later phases (P3–P7): + +- `SCRAPING_SUCCESSFUL` (21) — set by the ingest task when fresh scrape results with + `documents_scraped > 0` are ingested from S3. +- `TEST_INDEXING` (22) — in-flight: curated content is being indexed to OpenSearch test. +- `SCRAPING_FAILED` (23) — scrape produced zero documents, the SSM dispatch failed, or the + job stalled past the timeout. +- `INDEXING_FAILED_ON_TEST` (24) — the test-indexing run reported failure. +- `INDEXING_FAILED_ON_PROD` (25) — the prod-indexing run reported failure. +- `PRODUCTION_INDEXING` (26) — in-flight: curated content is being indexed to OpenSearch prod. + +Failure statuses (23–25) render `btn-danger`; in-flight statuses (22, 26) render `btn-light`. +Both Python colour maps fall back to `btn-light` for unmapped values instead of raising +`KeyError`. + ## Key Models and Files - `Collection`: Main model handling status transitions diff --git a/sde_collections/models/__init__.py b/sde_collections/models/__init__.py index e69de29b..143f16ae 100644 --- a/sde_collections/models/__init__.py +++ b/sde_collections/models/__init__.py @@ -0,0 +1,4 @@ +from .indexing import IndexDispatch +from .scraper_config import ScrapeDispatch, ScraperConfigOverride + +__all__ = ["IndexDispatch", "ScrapeDispatch", "ScraperConfigOverride"] diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index 097c5ce7..556efb06 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -1,28 +1,26 @@ # sde_collections/models/collection.py -import json -import urllib.parse - import requests from django.apps import apps +from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType -from django.db import models +from django.db import models, transaction from django.db.models.signals import post_save from django.dispatch import receiver from model_utils import FieldTracker from slugify import slugify -from config_generation.db_to_xml import XmlEditor from inference.models.inference_choice_fields import ( ClassificationType, InferenceJobStatus, ) from sde_collections.tasks import ( - fetch_full_text, + dispatch_scrape_job, + index_collection_to_prod, + index_collection_to_test, migrate_dump_to_delta_and_handle_status_transistions, ) -from ..utils.github_helper import GitHubHandler from ..utils.slack_utils import ( STATUS_CHANGE_NOTIFICATIONS, format_slack_message, @@ -41,7 +39,7 @@ from .delta_url import CuratedUrl, DeltaUrl, DumpUrl User = get_user_model() -DELTA_COMPARISON_FIELDS = ["scraped_title", "tdamm_tag", "division"] # Add more fields as needed +DELTA_COMPARISON_FIELDS = ["scraped_title", "scraped_text", "tdamm_tag", "division"] # Add more fields as needed # TODO: may need to double check how the ml fields are evaluated. we need to ensure that it looks # specifically at the ml value, not the default or the manual value. @@ -243,78 +241,10 @@ def promote_to_curated(self): # Step 4: Reapply patterns to DeltaUrls self.refresh_url_lists_for_all_patterns() - def add_to_public_query(self): - """Add the collection to the public query.""" - if self.workflow_status not in [ - WorkflowStatusChoices.QUALITY_CHECK_PERFECT, - WorkflowStatusChoices.QUALITY_CHECK_MINOR, - ]: - raise ValueError(f"{self.config_folder} is not ready for public prod, you can't add it to the public query") - - gh = GitHubHandler() - query_path = "webservices/query-smd-primary.xml" - scraper_content = gh._get_file_contents(query_path) - scraper_editor = XmlEditor(scraper_content.decoded_content.decode("utf-8")) - - collections = scraper_editor.get_tag_value("CollectionSelection", strict=True) - collections = collections.split(";") - collections.append(f"/SDE/{self.config_folder}/") - collections = list(set(collections)) - collections.sort() - collections = ";".join(collections) - - scraper_editor.update_or_add_element_value("CollectionSelection", collections) - scraper_content = scraper_editor.update_config_xml() - gh.create_or_update_file(query_path, scraper_content) - - @property - def _scraper_config_path(self) -> str: - return f"sources/scrapers/{self.config_folder}/default.xml" - - @property - def _indexer_config_path(self) -> str: - return f"sources/SDE/{self.config_folder}/default.xml" - - @property - def _indexer_job_path(self) -> str: - return f"jobs/collection.indexer.{self.config_folder}.xml" - - @property - def _scraper_job_path(self) -> str: - return f"jobs/collection.indexer.scrapers.{self.config_folder}.xml" - @property def tree_root(self) -> str: return f"/{self.get_division_display()}/{self.name}/" - @property - def server_url_secret_prod(self) -> str: - base_url = "https://sciencediscoveryengine.nasa.gov" # noqa: E231 - payload = { - "name": "secret-prod", - "scope": "All", - "text": "", - "advanced": { - "collection": f"/SDE/{self.config_folder}/", - }, - } - encoded_payload = urllib.parse.quote(json.dumps(payload)) - return f"{base_url}/app/secret-prod/#/search?query={encoded_payload}" - - @property - def server_url_prod(self) -> str: - base_url = "https://sciencediscoveryengine.nasa.gov" # noqa: E231 - payload = { - "name": "query-smd-primary", - "scope": "All", - "text": "", - "advanced": { - "collection": f"/SDE/{self.config_folder}/", - }, - } - encoded_payload = urllib.parse.quote(json.dumps(payload)) - return f"{base_url}/app/nasa-sba-smd/#/search?query={encoded_payload}" - @property def curation_status_button_color(self) -> str: color_choices = { @@ -354,9 +284,12 @@ def workflow_status_button_color(self) -> str: 20: "btn-info", 21: "btn-success", 22: "btn-light", - 23: "btn-light", + 23: "btn-danger", + 24: "btn-danger", + 25: "btn-danger", + 26: "btn-light", } - return color_choices[self.workflow_status] + return color_choices.get(self.workflow_status, "btn-light") @property def reindexing_status_button_color(self) -> str: @@ -371,131 +304,6 @@ def reindexing_status_button_color(self) -> str: } return color_choices[self.reindexing_status] - def _process_exclude_list(self): - """Process the exclude list.""" - return [pattern._process_match_pattern() for pattern in self.excludepattern.all()] - - def _process_include_list(self): - """Process the include list.""" - return [pattern._process_match_pattern() for pattern in self.includepattern.all()] - - def _process_title_list(self): - """Process the title list""" - title_rules = [] - for title_pattern in self.titlepattern.all(): - processed_pattern = { - "title_criteria": title_pattern._process_match_pattern(), - "title_value": title_pattern.title_pattern, - } - title_rules.append(processed_pattern) - return title_rules - - def _process_document_type_list(self): - """Process the document type list""" - document_type_rules = [] - for document_type_pattern in self.documenttypepattern.all(): - processed_pattern = { - "criteria": document_type_pattern._process_match_pattern(), - "document_type": document_type_pattern.get_document_type_display(), - } - document_type_rules.append(processed_pattern) - return document_type_rules - - def _write_to_github(self, path, content, overwrite): - gh = GitHubHandler() - if overwrite: - gh.create_or_update_file(path, content) - else: - gh.create_file(path, content) - - def create_scraper_config(self, overwrite: bool = False): - """ - Reads from the model data and creates the initial scraper config xml file - - if overwrite is True, it will overwrite the existing file - """ - - scraper_template = open("config_generation/xmls/scraper_template.xml").read() - editor = XmlEditor(scraper_template) - scraper_config = editor.convert_template_to_scraper(self) - self._write_to_github(self._scraper_config_path, scraper_config, overwrite) - - def create_indexer_config(self, overwrite: bool = False): - """ - Reads from the model data and creates the plugin config xml file that calls the api - - if overwrite is True, it will overwrite the existing file - """ - - # there needs to be a scraper config file before creating the plugin config - gh = GitHubHandler() - scraper_exists = gh.check_file_exists(self._scraper_config_path) - if not scraper_exists: - raise ValueError(f"Scraper does not exist for the collection {self.config_folder}") - else: - scraper_content = gh._get_file_contents(self._scraper_config_path) - scraper_content = scraper_content.decoded_content.decode("utf-8") - scraper_editor = XmlEditor(scraper_content) - - indexer_template = open("config_generation/xmls/indexer_template.xml").read() - indexer_editor = XmlEditor(indexer_template) - indexer_config = indexer_editor.convert_template_to_indexer(scraper_editor) - self._write_to_github(self._indexer_config_path, indexer_config, overwrite) - - def create_scraper_job(self, overwrite: bool = False): - """ - Reads from the model data and creates the initial scraper job xml file - - if overwrite is True, it will overwrite the existing file - """ - - scraper_job_template = open("config_generation/xmls/job_template.xml").read() - editor = XmlEditor(scraper_job_template) - scraper_job = editor.convert_template_to_job(self, "scrapers") - self._write_to_github(self._scraper_job_path, scraper_job, overwrite) - - def create_indexer_job(self, overwrite: bool = False): - """ - Reads from the model data and creates indexer job that calls the plugin config - - if overwrite is True, it will overwrite the existing file - """ - indexer_template = open("config_generation/xmls/job_template.xml").read() - editor = XmlEditor(indexer_template) - indexer_job = editor.convert_template_to_job(self, "SDE") - self._write_to_github(self._indexer_job_path, indexer_job, overwrite) - - def update_config_xml(self, original_config_string): - """ - reads from the model data and creates a config that mirrors the - - excludes - - title rules - - doc types - - tree root - """ - editor = XmlEditor(original_config_string) - - URL_EXCLUDES = self._process_exclude_list() - URL_INCLUDES = self._process_include_list() - TITLE_RULES = self._process_title_list() - DOCUMENT_TYPE_RULES = self._process_document_type_list() - - # TODO: this was creating duplicates so it was temporarily disabled - # if self.tree_root: - # editor.update_or_add_element_value("TreeRoot", self.tree_root) - - for url in URL_EXCLUDES: - editor.add_url_exclude(url) - for url in URL_INCLUDES: - editor.add_url_include(url) - for title_rule in TITLE_RULES: - editor.add_title_mapping(**title_rule) - for rule in DOCUMENT_TYPE_RULES: - editor.add_document_type_mapping(**rule) - - updated_config_xml_string = editor.update_config_xml() - return updated_config_xml_string - def _compute_config_folder_name(self) -> str: """ Take the human readable `self.name` and create a standardized machine format @@ -504,43 +312,6 @@ def _compute_config_folder_name(self) -> str: return slugify(self.name, separator="_") - def import_metadata_from_sinequa_config(self) -> bool: - """Import metadata from Sinequa.""" - if not self.config_folder: - return False - - gh = GitHubHandler(collections=[self]) - metadata = gh.fetch_metadata() - - try: - metadata[self.config_folder] - except KeyError: - return False - - print(f"Updating metadata for {self.name}") - # tree root - tree_root = metadata[self.config_folder]["tree_root"] - if tree_root != self.tree_root: - print(f"Updating tree root for {self.name} to {tree_root}") - self.tree_root = tree_root - - # document type - document_type = metadata[self.config_folder]["document_type"] - if document_type != self.document_type: - print(f"Updating document type for {self.name} to {document_type}") - self.document_type = document_type - - # connector - # connector = metadata[self.config_folder]["connector"] - # if connector != self.connector: - # print(f"Updating connector for {self.name} to {connector}") - # self.connector = connector - - self.save() - print("\n\n") - - return True - def __str__(self) -> str: """Unicode representation of Collection.""" return self.name @@ -553,11 +324,6 @@ def has_folder(self) -> bool: def candidate_urls_count(self) -> int: return self.candidate_urls.count() - @property - def sinequa_configuration(self) -> str: - URL = f"https://github.com/NASA-IMPACT/sde-backend/blob/production/sources/SDE/{self.config_folder}/default.xml" # noqa: E231, E501 - return URL - @property def github_issue_link(self) -> str: return f"https://github.com/NASA-IMPACT/sde-project/issues/{self.github_issue_number}" # noqa: E231 @@ -691,6 +457,13 @@ def generate_inference_job(self, classification_type): def queue_necessary_classifications(self): """Check if collection needs classification and queue jobs if needed""" + if not settings.INFERENCE_ENABLED: + # Inference pipeline is dormant: never create an InferenceJob (it would queue + # forever and strand the collection before Ready for Curation) — go straight + # to migration for every collection, including the TDAMM-listed ones. + migrate_dump_to_delta_and_handle_status_transistions.delay(self.id) + return + tdamm_collections = [ "imagine_the_universe", "physics_of_the_cosmos", @@ -730,22 +503,9 @@ def save(self, *args, **kwargs): if not self.config_folder: self.config_folder = self._compute_config_folder_name() - if not self._state.adding: - old_status = Collection.objects.get(id=self.id).workflow_status - new_status = self.workflow_status - if old_status != new_status: - transition = (old_status, new_status) - if transition in STATUS_CHANGE_NOTIFICATIONS: - details = STATUS_CHANGE_NOTIFICATIONS[transition] - message = format_slack_message(self.name, details, self.id) - try: - # TODO: find a better way to allow this to work on dev environments with - # no slack integration - send_slack_message(message) - except Exception as e: - print(f"Error sending Slack message: {e}") - - # Call the parent class's save method + # Status-change Slack notifications live in the post_save receiver + # (handle_workflow_status_change), so a message can never be sent for a save + # that then fails. super().save(*args, **kwargs) def __init__(self, *args, **kwargs): @@ -818,9 +578,12 @@ def workflow_status_button_color(self) -> str: 20: "btn-info", 21: "btn-success", 22: "btn-light", - 23: "btn-light", + 23: "btn-danger", + 24: "btn-danger", + 25: "btn-danger", + 26: "btn-light", } - return color_choices[self.workflow_status] + return color_choices.get(self.workflow_status, "btn-light") @receiver(post_save, sender=Collection) @@ -868,10 +631,43 @@ def reindexing_status_button_color(self) -> str: return color_choices[self.reindexing_status] -@receiver(post_save, sender=Collection) -def create_configs_on_status_change(sender, instance, created, **kwargs): - """Creates various config files on certain workflow status changes""" +def _enqueue_on_commit(task, collection_id): + """Enqueue `task` once the surrounding transaction commits (immediately when there + is none). Keeps post_save side effects from racing uncommitted writes.""" + transaction.on_commit(lambda: task.delay(collection_id)) + + +def _send_status_change_notification(instance, old_status, new_status): + """Slack notification for a committed status transition. Lives in post_save (not + Collection.save) so a message can never be sent for a save that then fails, and so + it reuses the tracker's old value instead of an extra Collection.objects.get().""" + details = STATUS_CHANGE_NOTIFICATIONS.get((old_status, new_status)) + if details is None: + return + message = format_slack_message(instance.name, details, instance.id) + try: + # TODO: find a better way to allow this to work on dev environments with + # no slack integration + send_slack_message(message) + except Exception as e: + print(f"Error sending Slack message: {e}") + +@receiver(post_save, sender=Collection) +def handle_workflow_status_change(sender, instance, created, **kwargs): + """Single dispatcher for status-triggered side effects (WORKFLOW.md steps 12–18). + + workflow_status: + READY_FOR_ENGINEERING -> dispatch_scrape_job (P3) + CURATED -> promote_to_curated + test-indexing hand-off + QC_PERFECT / QC_MINOR -> prod-indexing hand-off + reindexing_status: + REINDEXING_NEEDED_ON_DEV -> dispatch_scrape_job (P3, re-scrape) + REINDEXING_CURATED -> promote_to_curated + + REINDEXING_FINISHED_ON_DEV deliberately triggers nothing: the P4 ingest sets that + status itself, so a trigger here would double-fire. + """ if getattr(instance, "_handling_status_change", False): return @@ -879,25 +675,27 @@ def create_configs_on_status_change(sender, instance, created, **kwargs): instance._handling_status_change = True if "workflow_status" in instance.tracker.changed(): - if instance.workflow_status == WorkflowStatusChoices.READY_FOR_CURATION: - instance.create_indexer_config(overwrite=True) - instance.create_indexer_job(overwrite=False) + old_status = instance.tracker.changed()["workflow_status"] + _send_status_change_notification(instance, old_status, instance.workflow_status) + + # Tasks are enqueued on commit: admin change views and callers wrapping the + # save in transaction.atomic() would otherwise let the worker start before + # the status (and promote_to_curated's CuratedUrl rows) are visible to it. + if instance.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING: + _enqueue_on_commit(dispatch_scrape_job, instance.id) elif instance.workflow_status == WorkflowStatusChoices.CURATED: instance.promote_to_curated() - elif instance.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING: - instance.create_scraper_config(overwrite=False) - instance.create_scraper_job(overwrite=False) - elif instance.workflow_status == WorkflowStatusChoices.INDEXING_FINISHED_ON_DEV: - fetch_full_text.delay(instance.id, "lrm_dev") + _enqueue_on_commit(index_collection_to_test, instance.id) elif instance.workflow_status in [ WorkflowStatusChoices.QUALITY_CHECK_PERFECT, WorkflowStatusChoices.QUALITY_CHECK_MINOR, ]: - instance.add_to_public_query() + _enqueue_on_commit(index_collection_to_prod, instance.id) if "reindexing_status" in instance.tracker.changed(): - if instance.reindexing_status == ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV: - fetch_full_text.delay(instance.id, "lrm_dev") + if instance.reindexing_status == ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV: + # Re-scrape path: replaces the engineer manually re-running a Sinequa job. + _enqueue_on_commit(dispatch_scrape_job, instance.id) elif instance.reindexing_status == ReindexingStatusChoices.REINDEXING_CURATED: instance.promote_to_curated() diff --git a/sde_collections/models/collection_choice_fields.py b/sde_collections/models/collection_choice_fields.py index a433317a..7f0ab8dd 100644 --- a/sde_collections/models/collection_choice_fields.py +++ b/sde_collections/models/collection_choice_fields.py @@ -98,6 +98,12 @@ class WorkflowStatusChoices(models.IntegerChoices): MERGE_PENDING = 17, "Code Merge Pending" NEEDS_DELETE = 19, "Delete from Prod" INDEXING_FINISHED_ON_DEV = 20, "Indexing Finished on LRM Dev" + SCRAPING_SUCCESSFUL = 21, "Scraping Successful" + TEST_INDEXING = 22, "Test Indexing" + SCRAPING_FAILED = 23, "Scraping Failed" + INDEXING_FAILED_ON_TEST = 24, "Indexing Failed on Test" + INDEXING_FAILED_ON_PROD = 25, "Indexing Failed on Prod" + PRODUCTION_INDEXING = 26, "Production Indexing" class ReindexingStatusChoices(models.IntegerChoices): diff --git a/sde_collections/models/indexing.py b/sde_collections/models/indexing.py new file mode 100644 index 00000000..dfdb33ec --- /dev/null +++ b/sde_collections/models/indexing.py @@ -0,0 +1,38 @@ +from django.db import models + + +class IndexDispatch(models.Model): + """One row per WEB_COSMOS indexing dispatch (test or prod target). + + Required because CELERY_RESULT_BACKEND = None means Celery task state cannot be + polled: dispatched_at is the stall-timeout reference (mirroring ScrapeDispatch), and + the COSMOS-minted run_id namespaces every S3 artifact, so an old run's status.json + can never satisfy a newer dispatch — no LastModified freshness rule needed. + """ + + TARGET_TEST = "test" + TARGET_PROD = "prod" + TARGET_CHOICES = [(TARGET_TEST, TARGET_TEST), (TARGET_PROD, TARGET_PROD)] + + collection = models.ForeignKey( + "sde_collections.Collection", + on_delete=models.CASCADE, + related_name="index_dispatches", + ) + run_id = models.CharField(max_length=64) + target = models.CharField(max_length=8, choices=TARGET_CHOICES) + task_arn = models.CharField(max_length=256, blank=True, default="") + # The workflow status the collection entered the dispatch with — a succeeded prod run + # mirrors QC_PERFECT/QC_MINOR onto PROD_PERFECT/PROD_MINOR (WORKFLOW.md step 30). + previous_workflow_status = models.IntegerField(null=True, blank=True) + dispatched_at = models.DateTimeField(auto_now_add=True) + # Set when the poller resolves the run (success, failure, or stall); resolved rows + # are never polled again, which also keeps the Slack report to a single post. + completed_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ["-dispatched_at"] + verbose_name_plural = "Index Dispatches" + + def __str__(self): + return f"{self.collection.config_folder} -> {self.target} ({self.run_id})" diff --git a/sde_collections/models/scraper_config.py b/sde_collections/models/scraper_config.py new file mode 100644 index 00000000..aaf34e6e --- /dev/null +++ b/sde_collections/models/scraper_config.py @@ -0,0 +1,50 @@ +from django.db import models + + +class ScraperConfigOverride(models.Model): + """Per-collection overrides merged onto the crawler's own defaults. + + All fields nullable: only non-null values are emitted into the job JSON, + because crawl4ai's merge_job() skips None. Curators edit these in the admin + console (WORKFLOW.md step 6). + """ + + collection = models.OneToOneField( + "sde_collections.Collection", + on_delete=models.CASCADE, + related_name="scraper_config", + ) + max_pages = models.PositiveIntegerField(null=True, blank=True, help_text="Crawler cap: 100,000") + depth_limit = models.PositiveIntegerField(null=True, blank=True) + delay = models.FloatField(null=True, blank=True, help_text="Seconds between requests (crawler default 0.25)") + concurrent_requests = models.PositiveSmallIntegerField(null=True, blank=True) + obey_robots = models.BooleanField(null=True, blank=True) + include_subdomains = models.BooleanField(null=True, blank=True) + + class Meta: + verbose_name = "Scraper Config Override" + + def __str__(self): + return f"Scraper overrides for {self.collection.config_folder}" + + +class ScrapeDispatch(models.Model): + """One row per SSM dispatch. Two jobs: (1) give the poller a freshness reference — + S3 results older than dispatched_at belong to a previous run and must be ignored; + (2) give the stall timeout a start time. Never deleted; latest row per collection wins. + """ + + collection = models.ForeignKey( + "sde_collections.Collection", + on_delete=models.CASCADE, + related_name="scrape_dispatches", + ) + dispatched_at = models.DateTimeField(auto_now_add=True) + ssm_command_id = models.CharField(max_length=64) + + class Meta: + ordering = ["-dispatched_at"] + verbose_name_plural = "Scrape Dispatches" + + def __str__(self): + return f"{self.collection.config_folder} @ {self.dispatched_at:%Y-%m-%d %H:%M:%S} ({self.ssm_command_id})" diff --git a/config_generation/tests/__init__.py b/sde_collections/scraping/__init__.py similarity index 100% rename from config_generation/tests/__init__.py rename to sde_collections/scraping/__init__.py diff --git a/sde_collections/scraping/job_builder.py b/sde_collections/scraping/job_builder.py new file mode 100644 index 00000000..861627d3 --- /dev/null +++ b/sde_collections/scraping/job_builder.py @@ -0,0 +1,38 @@ +"""Build the job JSON the crawl4ai crawler consumes. + +Contract (sde-crawl4ai-scraper-v1/sde_crawler/job.py): merge_job() skips None values, +so only non-null overrides are emitted; `collection_id` names both the output file and +the S3 keys. COSMOS always sends config_folder — unique, editable=False, stable across +renames. Never the numeric PK. +""" + +# Mirrors sde_crawler.crawler.MAX_PAGES_CAP — jobs above this land in jobs/failed/. +MAX_PAGES_CAP = 100_000 + +OVERRIDE_FIELDS = ( + "max_pages", + "depth_limit", + "delay", + "concurrent_requests", + "obey_robots", + "include_subdomains", +) + + +def build_job_json(collection) -> dict: + job = {"seed": collection.url, "collection_id": collection.config_folder} + + override = collection.scraper_config if hasattr(collection, "scraper_config") else None + if override is not None: + for field in OVERRIDE_FIELDS: + value = getattr(override, field) + if value is not None: + job[field] = value + + if "max_pages" in job and job["max_pages"] > MAX_PAGES_CAP: + raise ValueError( + f"max_pages={job['max_pages']} exceeds the crawler cap of {MAX_PAGES_CAP}; " + f"the job would be rejected into jobs/failed/" + ) + + return job diff --git a/sde_collections/scraping/s3_results.py b/sde_collections/scraping/s3_results.py new file mode 100644 index 00000000..d7b0d027 --- /dev/null +++ b/sde_collections/scraping/s3_results.py @@ -0,0 +1,65 @@ +"""Read crawl results from the crawler's S3 bucket. + +Key layout mirrors sde-crawl4ai-scraper-v1/sde_crawler/job.py::s3_keys_for_collection — +do not reinvent it. The crawler writes no status file: the failures summary is written +only at the end of a completed run, so a *fresh* summary is the completion marker. +S3 objects persist between runs, which is why every check must be filtered against the +latest ScrapeDispatch.dispatched_at — without it, any re-dispatch would instantly +"complete" against the previous run's output. +""" + +import json + +from botocore.exceptions import ClientError +from django.conf import settings + +from ..utils.aws import get_boto3_session + +_MISSING_KEY_CODES = {"NoSuchKey", "404"} + + +def s3_keys_for_collection(collection_id: str) -> dict[str, str]: + return { + "documents": f"scraped_collections/{collection_id}.json", + "failures": f"failure_logs/{collection_id}_failures.jsonl", + "summary": f"failure_logs/{collection_id}_failures_summary.json", + } + + +def _get_object(key: str): + s3 = get_boto3_session().client("s3") + return s3.get_object(Bucket=settings.SDE_S3_BUCKET, Key=key) + + +def fetch_summary(collection_id: str): + """Return (summary_dict, last_modified) or None if no summary object exists.""" + try: + obj = _get_object(s3_keys_for_collection(collection_id)["summary"]) + except ClientError as e: + if e.response.get("Error", {}).get("Code") in _MISSING_KEY_CODES: + return None + raise + return json.loads(obj["Body"].read()), obj["LastModified"] + + +def fetch_documents(collection_id: str) -> list[dict]: + """The scraped documents array; each item has exactly seven fields: + {url, title, full_text, content_type, seed, host, depth}.""" + obj = _get_object(s3_keys_for_collection(collection_id)["documents"]) + return json.loads(obj["Body"].read()) + + +def results_ready(collection_id: str, dispatched_at): + """Return the summary dict if a run completed *after* dispatched_at, else None. + + A summary whose LastModified predates the dispatch belongs to a previous run and is + treated as absent. dispatched_at=None (no dispatch recorded) accepts any summary — + that path is reserved for explicit manual ingest. + """ + found = fetch_summary(collection_id) + if found is None: + return None + summary, last_modified = found + if dispatched_at is not None and last_modified <= dispatched_at: + return None + return summary diff --git a/sde_collections/scraping/ssm_dispatch.py b/sde_collections/scraping/ssm_dispatch.py new file mode 100644 index 00000000..da16231a --- /dev/null +++ b/sde_collections/scraping/ssm_dispatch.py @@ -0,0 +1,39 @@ +"""Deliver a job JSON into the crawler's inbox on EC2 via SSM. + +Mirrors sde-crawl4ai-scraper-v1/scripts/drop_job.sh: AWS-RunShellScript writing to +{CRAWLER_INBOX_PATH}/{config_folder}.json, chowned to ec2-user. The JSON is passed +through shlex.quote — seed URLs contain characters that would break a heredoc — and +written to a .tmp path first so the inbox watcher can never read a partial file. +""" + +import json +import shlex + +from django.conf import settings + +from ..utils.aws import get_boto3_session +from .job_builder import build_job_json + + +def send_job_to_crawler(collection) -> str: + """Build and drop the job JSON for `collection`; returns the SSM command id.""" + for name in ("CRAWLER_INSTANCE_ID", "CRAWLER_INBOX_PATH"): + if not getattr(settings, name): + raise ValueError(f"{name} is not configured — cannot dispatch a scrape job") + + job_json = json.dumps(build_job_json(collection)) + dest = f"{settings.CRAWLER_INBOX_PATH}/{collection.config_folder}.json" + script = ( + f"printf '%s' {shlex.quote(job_json)} > {shlex.quote(dest + '.tmp')}\n" + f"chown ec2-user:ec2-user {shlex.quote(dest + '.tmp')}\n" + f"mv -f {shlex.quote(dest + '.tmp')} {shlex.quote(dest)}\n" + ) + + ssm = get_boto3_session().client("ssm") + response = ssm.send_command( + InstanceIds=[settings.CRAWLER_INSTANCE_ID], + DocumentName="AWS-RunShellScript", + Comment=f"COSMOS scrape dispatch: {collection.config_folder}"[:100], + Parameters={"commands": [script]}, + ) + return response["Command"]["CommandId"] diff --git a/sde_collections/signals.py b/sde_collections/signals.py new file mode 100644 index 00000000..aaad28dd --- /dev/null +++ b/sde_collections/signals.py @@ -0,0 +1,49 @@ +from django.conf import settings +from django.db.models.signals import post_migrate +from django.dispatch import receiver + +POLL_SCRAPE_TASK = "sde_collections.tasks.poll_scrape_jobs" +POLL_SCRAPE_TASK_NAME = "Poll crawler S3 results (every 5 min)" +POLL_INDEX_TASK = "sde_collections.tasks.poll_index_runs" +POLL_INDEX_TASK_NAME = "Poll index runs (every 2 min)" + + +def _ensure_beat_row(name, task_path, crontab, enabled): + from django_celery_beat.models import PeriodicTask + + try: + task = PeriodicTask.objects.get(name=name) + except PeriodicTask.DoesNotExist: + PeriodicTask.objects.create(crontab=crontab, name=name, task=task_path, enabled=enabled) + else: + task.crontab = crontab + task.task = task_path + task.enabled = enabled + task.save() + + +@receiver(post_migrate) +def create_periodic_tasks(sender, **kwargs): + """DB-row beat schedules for the S3 pollers (there is no CELERY_BEAT_SCHEDULE in + this repo — all schedules are django-celery-beat rows). + + `enabled` is re-asserted from the flags on every migrate, same pattern as + inference/signals.py: the flag, not a hand-edit in the admin, is the source of truth. + """ + if sender.name != "sde_collections": + return + + from django_celery_beat.models import CrontabSchedule + + def crontab_every(minutes): + crontab, _ = CrontabSchedule.objects.get_or_create( + minute=f"*/{minutes}", + hour="*", + day_of_week="*", + day_of_month="*", + month_of_year="*", + ) + return crontab + + _ensure_beat_row(POLL_SCRAPE_TASK_NAME, POLL_SCRAPE_TASK, crontab_every(5), settings.SCRAPE_POLL_ENABLED) + _ensure_beat_row(POLL_INDEX_TASK_NAME, POLL_INDEX_TASK, crontab_every(2), settings.INDEX_POLL_ENABLED) diff --git a/sde_collections/sinequa_api.py b/sde_collections/sinequa_api.py deleted file mode 100644 index e1782ce8..00000000 --- a/sde_collections/sinequa_api.py +++ /dev/null @@ -1,324 +0,0 @@ -import json -from collections.abc import Iterator -from typing import Any - -import requests -import urllib3 -from django.conf import settings - -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -server_configs = { - "dev": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://sde-renaissance.nasa-impact.net", - }, - "test": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.test.nasa.gov", - "index": "sde_index", - }, - "production": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.nasa.gov", - "index": "sde_index", - }, - "secret_test": { - "app_name": "nasa-sba-sde", - "query_name": "query-sde-primary", - "base_url": "https://sciencediscoveryengine.test.nasa.gov", - "index": "sde_index", - }, - "secret_production": { - "app_name": "nasa-sba-sde", - "query_name": "query-sde-primary", - "base_url": "https://sciencediscoveryengine.nasa.gov", - "index": "sde_index", - }, - "xli": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://sde-xli.nasa-impact.net", - "index": "sde_index", - }, - "lrm_dev": { - "app_name": "sde-init-check", - "query_name": "query-init-check", - "base_url": "https://sde-lrm.nasa-impact.net", - "index": "sde_init_check", - }, - "lrm_qa": { - "app_name": "sde-init-check", - "query_name": "query-init-check", - "base_url": "https://sde-qa.nasa-impact.net", - }, -} - - -class Api: - def __init__(self, server_name: str = None, user: str = None, password: str = None, token: str = None) -> None: - self.server_name = server_name - if server_name not in server_configs: - raise ValueError(f"Invalid server configuration: '{server_name}' is not a recognized server name") - - self.config = server_configs[server_name] - self.app_name: str = self.config["app_name"] - self.query_name: str = self.config["query_name"] - self.base_url: str = self.config["base_url"] - self.dev_servers = ["xli", "lrm_dev", "lrm_qa"] - - self._provided_user = user - self._provided_password = password - self._provided_token = token - - def _get_user(self) -> str | None: - """Retrieve the user, using the provided value or defaulting to Django settings.""" - return self._provided_user or getattr(settings, f"{self.server_name}_USER".upper(), None) - - def _get_password(self) -> str | None: - """Retrieve the password, using the provided value or defaulting to Django settings.""" - return self._provided_password or getattr(settings, f"{self.server_name}_PASSWORD".upper(), None) - - def _get_token(self) -> str | None: - """Retrieve the token, using the provided value or defaulting to Django settings.""" - return self._provided_token or getattr(settings, f"{self.server_name}_TOKEN".upper(), None) - - def _get_source_name(self) -> str: - """by default, the source is /SDE/. However for the various dev servers, the source is tends to be /scrapers/""" - return "scrapers" if self.server_name in self.dev_servers else "SDE" - - def process_response( - self, - url: str, - payload: dict[str, Any] | None = None, - headers: dict[str, str] | None = None, - raw_data: str | None = None, - ) -> Any: - """Sends a POST request and processes the response.""" - response = requests.post( - url, headers=headers, json=payload if raw_data is None else None, data=raw_data, verify=False - ) - if response.status_code == requests.codes.ok: - return response.json() - else: - response.raise_for_status() - - def query(self, page: int, collection_config_folder: str | None = None, source: str | None = None) -> Any: - url = f"{self.base_url}/api/v1/search.query" - if self.server_name in self.dev_servers: - user = self._get_user() - password = self._get_password() - if not user or not password: - raise ValueError( - f"Authentication error: Missing credentials for dev server '{self.server_name}'. " - f"Both username and password are required for servers: {', '.join(self.dev_servers)}" - ) - authentication = f"?Password={password}&User={user}" - url = f"{url}{authentication}" - - payload = { - "app": self.app_name, - "query": { - "name": self.query_name, - "text": "", - "page": page, - "pageSize": 1000, - "advanced": {}, - }, - } - - if collection_config_folder: - source = source if source else self._get_source_name() - payload["query"]["advanced"]["collection"] = f"/{source}/{collection_config_folder}/" - - return self.process_response(url, payload) - - def _execute_sql_query(self, sql: str) -> dict: - """ - Executes a SQL query against the Sinequa API. - - Args: - sql (str): The SQL query to execute - - Returns: - dict: The JSON response from the API containing 'Rows' and 'TotalRowCount' - - Raises: - ValueError: If no token is available for authentication - """ - token = self._get_token() - if not token: - raise ValueError("Authentication error: Token is required for SQL endpoint access") - - url = f"{self.base_url}/api/v1/engine.sql" - headers = {"Content-Type": "application/json", "Authorization": f"Bearer {token}"} - raw_payload = json.dumps( - { - "method": "engine.sql", - "sql": sql, - "pretty": True, - } - ) - - return self.process_response(url, headers=headers, raw_data=raw_payload) - - def _process_rows_to_records(self, rows: list) -> list[dict]: - """ - Converts raw SQL row data into structured record dictionaries. - - Args: - rows (list): List of rows, where each row is [url, full_text, title] - - Returns: - list[dict]: List of processed records with url, full_text, and title keys - - Raises: - ValueError: If any row doesn't contain exactly 3 elements - """ - processed_records = [] - for idx, row in enumerate(rows): - if len(row) != 3: - raise ValueError( - f"Invalid row format at index {idx}: Expected exactly three elements (url, full_text, title). " - f"Received {len(row)} elements." - ) - processed_records.append({"url": row[0], "full_text": row[1], "title": row[2]}) - return processed_records - - def get_full_texts( - self, - collection_config_folder: str, - source: str = None, - start_at: int = 0, - batch_size: int = 500, - min_batch_size: int = 1, - ) -> Iterator[dict]: - """ - Retrieves and yields batches of text records from the SQL database for a given collection. - Uses pagination to handle large datasets efficiently. If a query fails, it automatically - reduces the batch size and retries, with the ability to recover batch size after successful queries. - - Args: - collection_config_folder (str): The collection folder to query (e.g., "EARTHDATA", "CASEI") - source (str, optional): The source to query. If None, defaults to "scrapers" for dev servers - or "SDE" for other servers. - start_at (int, optional): Starting offset for records. Defaults to 0. - page_size (int, optional): Initial number of records per batch. Defaults to 500. - min_batch_size (int, optional): Minimum batch size before giving up. Defaults to 1. - - Yields: - list[dict]: Batches of records, where each record is a dictionary containing: - { - "url": str, # The URL of the document - "full_text": str, # The full text content of the document - "title": str # The title of the document - } - - Raises: - ValueError: If the server's index is not defined in its configuration - ValueError: If batch size reaches minimum without success - - Note: - - Results are paginated with adaptive batch sizing - - Each batch is processed into clean dictionaries before being yielded - - The iterator will stop when either: - 1. No more rows are returned from the query - 2. The total count of records has been reached - - Batch size will decrease on failure and can recover after successful queries - """ - if not source: - source = self._get_source_name() - - if (index := self.config.get("index")) is None: - raise ValueError( - f"Configuration error: Index not defined for server '{self.server_name}'. " - "Please update server configuration with the required index." - ) - - base_sql = f"SELECT url1, text, title FROM {index} WHERE collection = '/{source}/{collection_config_folder}/'" - - current_offset = start_at - current_batch_size = batch_size - total_count = None - - while True: - sql = f"{base_sql} SKIP {current_offset} COUNT {current_batch_size}" - - try: - response = self._execute_sql_query(sql) - rows = response.get("Rows", []) - - if not rows: # Stop if we get an empty batch - break - - if total_count is None: - total_count = response.get("TotalRowCount", 0) - - yield (self._process_rows_to_records(rows)) - - current_offset += len(rows) - - if total_count and current_offset >= total_count: # Stop if we've processed all records - break - - except (requests.RequestException, ValueError) as e: - if current_batch_size <= min_batch_size: - raise ValueError( - f"Failed to process batch even at minimum size {min_batch_size}. " f"Last error: {str(e)}" - ) - - # Halve the batch size and retry - current_batch_size = max(current_batch_size // 2, min_batch_size) - print(f"Reducing batch size to {current_batch_size} and retrying...") - continue - - def get_total_count(self, collection_config_folder: str, source: str = None) -> int: - """ - Retrieves the total count of records for a given collection using Sinequa's TotalRowCount metadata. - - Args: - collection_config_folder (str): The collection folder to query (e.g., "EARTHDATA", "CASEI"). - source (str, optional): The source to query. If None, defaults to "scrapers" for dev servers - or "SDE" for other servers. - - Returns: - int: The total number of records in the collection. - """ - if not source: - source = self._get_source_name() - - if (index := self.config.get("index")) is None: - raise ValueError( - f"Configuration error: Index not defined for server '{self.server_name}'. " - "Please update server configuration with the required index." - ) - - # Minimal query to get only metadata, no data retrieval - sql = f"SELECT * FROM {index} WHERE collection = '/{source}/{collection_config_folder}/' SKIP 0 COUNT 0" - - response = self._execute_sql_query(sql) - - # Extract TotalRowCount from metadata - return response.get("TotalRowCount", 0) - - @staticmethod - def _process_full_text_response(batch_data: dict): - if "Rows" not in batch_data or not isinstance(batch_data["Rows"], list): - raise ValueError( - "Invalid response format: Expected 'Rows' key with list data in Sinequa server response. " - f"Received: {type(batch_data.get('Rows', None))}" - ) - - processed_data = [] - for idx, row in enumerate(batch_data["Rows"]): - if len(row) != 3: - raise ValueError( - f"Invalid row format at index {idx}: Expected exactly three elements (url, full_text, title). " - f"Received {len(row)} elements." - ) - url, full_text, title = row - processed_data.append({"url": url, "full_text": full_text, "title": title}) - return processed_data diff --git a/sde_collections/tasks.py b/sde_collections/tasks.py index 98f26a34..4caa34ba 100644 --- a/sde_collections/tasks.py +++ b/sde_collections/tasks.py @@ -1,13 +1,9 @@ # /sde_collections/tasks.py -import json -import os -import shutil -import boto3 +import logging + from django.apps import apps from django.conf import settings -from django.core import management -from django.core.management.commands import loaddata from django.db import transaction from config import celery_app @@ -16,106 +12,72 @@ WorkflowStatusChoices, ) +from .indexing.dispatch import run_index_task +from .indexing.export import export_curated_to_s3 +from .indexing.run_status import fetch_run_status, fetch_validation_report from .models.delta_url import DumpUrl -from .sinequa_api import Api -from .utils.github_helper import GitHubHandler - - -def _get_data_to_import(collection, server_name): - # ignore these because they are API collections and don't have URLs - ignore_collections = [ - "/SMD/ASTRO_NAVO_HEASARC/", - "/SMD/CASEI_Campaign/", - "/SMD/CASEI_Deployment/", - "/SMD/CASEI_Instrument/", - "/SMD/CASEI_Platform/", - "/SMD/CMR_API/", - "/SMD/PDS_API_Legacy_All/", - ] - - data_to_import = [] - api = Api(server_name=server_name) - page = 1 - while True: - print(f"Getting page: {page}") - response = api.query(page=page, collection_config_folder=collection.config_folder) - if response["cursorRowCount"] == 0: - break - - for record in response.get("records", []): - full_collection_name = record.get("collection")[0] - if full_collection_name in ignore_collections: - continue - - url = record.get("download_url") - title = record.get("title", "") - collection_pk = collection.pk - - if not url: - continue - - augmented_data = { - "model": "sde_collections.url", - "fields": { - "collection": collection_pk, - "url": url, - "scraped_title": title, - }, - } - - data_to_import.append(augmented_data) - page += 1 - return data_to_import - - -@celery_app.task(soft_time_limit=10000) -def import_candidate_urls_from_api(server_name="test", collection_ids=[]): - TEMP_FOLDER_NAME = "temp" - os.makedirs(TEMP_FOLDER_NAME, exist_ok=True) - Collection = apps.get_model("sde_collections", "Collection") - - collections = Collection.objects.filter(id__in=collection_ids) - - for collection in collections: - urls_file = f"{TEMP_FOLDER_NAME}/{collection.config_folder}.json" - - print("Getting responses from API") - data_to_import = _get_data_to_import(server_name=server_name, collection=collection) - print(f"Got {len(data_to_import)} records for {collection.config_folder}") - - print("Dumping django fixture to file") - json.dump(data_to_import, open(urls_file, "w")) - - print("Deleting existing candidate URLs") - # this sometimes takes a while - collection.candidate_urls.all().delete() +from .models.indexing import IndexDispatch +from .scraping.s3_results import fetch_documents, fetch_summary, results_ready +from .scraping.ssm_dispatch import send_job_to_crawler +from .utils.slack_utils import ( + notify_status_change, + send_detailed_import_notification, + send_indexing_validation_report, + send_slack_message, +) - print("Loading fixture; this may take a while") - # subprocess.call(f'python manage.py loaddata "{urls_file}"', shell=True) - management.call_command(loaddata.Command(), urls_file) +logger = logging.getLogger(__name__) - print("Applying existing patterns; this may take a while") - collection.apply_all_patterns() +# Statuses a collection can hold while a crawl is in flight on the *workflow* path. A +# scrape/ingest failure may only rewrite workflow_status when it is one of these; on the +# re-scrape path (reindexing_status) the live workflow status — typically PROD_* — is +# left alone. +SCRAPE_FLOW_STATUSES = ( + WorkflowStatusChoices.READY_FOR_ENGINEERING, + WorkflowStatusChoices.ENGINEERING_IN_PROGRESS, + WorkflowStatusChoices.SCRAPING_SUCCESSFUL, +) - if collection.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING: - collection.workflow_status = WorkflowStatusChoices.ENGINEERING_IN_PROGRESS - collection.save() - # Finally set the status to READY_FOR_CURATION - collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION - collection.save() +def _record_status_transition(collection, old_status, new_status): + """Side effects of a workflow transition made via queryset .update() rather than + save(): the history row and Slack post that post_save would otherwise produce. + Tasks update this way on purpose — as a compare-and-swap claim, and so the + dispatch trigger can't re-fire — which is exactly why the timeline row must be + written here.""" + WorkflowHistory = apps.get_model("sde_collections", "WorkflowHistory") + WorkflowHistory.objects.create( + collection=collection, + workflow_status=new_status, + curated_by=collection.curated_by, + old_status=old_status, + ) + notify_status_change(collection.name, collection.id, old_status, new_status) - print("Deleting temp files") - shutil.rmtree(TEMP_FOLDER_NAME) +def _mark_scrape_failed(collection, reason): + """Record a scrape/ingest failure without clobbering a status outside the scrape flow. -@celery_app.task() -def push_to_github_task(collection_ids): + Uses a queryset .update() (so no post_save side effects fire) and posts the mapped + Slack message itself. Outside the scrape flow only an alert is sent. + """ Collection = apps.get_model("sde_collections", "Collection") - - collections = Collection.objects.filter(id__in=collection_ids) - github_handler = GitHubHandler(collections) - github_handler.push_to_github() + old_status = Collection.objects.filter(id=collection.id).values_list("workflow_status", flat=True).first() + if old_status in SCRAPE_FLOW_STATUSES: + Collection.objects.filter(id=collection.id).update(workflow_status=WorkflowStatusChoices.SCRAPING_FAILED) + _record_status_transition(collection, old_status, WorkflowStatusChoices.SCRAPING_FAILED) + return + # Re-scrape path: keep the live workflow status, and clear the reindexing request so + # the poller stops re-enqueueing the same failed run every 5 minutes. + logger.error("Re-scrape of %s failed (%s); workflow status left unchanged", collection.config_folder, reason) + Collection.objects.filter(id=collection.id).update(reindexing_status=ReindexingStatusChoices.REINDEXING_NOT_NEEDED) + try: + send_slack_message( + f"Alert: re-scrape of '{collection.name}' failed ({reason}). " + f"Workflow status was left unchanged; set Re-Indexing Needed again to retry." + ) + except Exception as e: + logger.warning("Error sending Slack message for %s: %s", collection.name, e) @celery_app.task() @@ -126,29 +88,6 @@ def sync_with_production_webapp(): collection.sync_with_production_webapp() -@celery_app.task() -def pull_latest_collection_metadata_from_github(): - Collection = apps.get_model("sde_collections", "Collection") - - FILENAME = "github_collections.json" - - gh = GitHubHandler(collections=Collection.objects.none()) - collections = gh.get_collections_from_github() - - json.dump(collections, open(FILENAME, "w"), indent=4) - - # Upload the file to S3 - s3_bucket_name = settings.AWS_STORAGE_BUCKET_NAME - s3_key = FILENAME - s3_client = boto3.client( - "s3", - region_name="us-east-1", - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, - ) - s3_client.upload_file(FILENAME, s3_bucket_name, s3_key) - - @celery_app.task() def resolve_title_pattern(title_pattern_id): TitlePattern = apps.get_model("sde_collections", "TitlePattern") @@ -156,47 +95,6 @@ def resolve_title_pattern(title_pattern_id): title_pattern.apply() -@celery_app.task(soft_time_limit=600) -def fetch_full_text(collection_id, server_name): - """Task to fetch full text and create DumpUrls only (no migration)""" - Collection = apps.get_model("sde_collections", "Collection") - collection = Collection.objects.get(id=collection_id) - api = Api(server_name) - - # Step 1: Delete existing DumpUrl entries - deleted_count, _ = DumpUrl.objects.filter(collection=collection).delete() - print(f"Deleted {deleted_count} old records.") - try: - total_server_count = api.get_total_count(collection.config_folder) - print(f"Total records on the server: {total_server_count}") - - # Step 2: Process data in batches - total_processed = 0 - for batch in api.get_full_texts(collection.config_folder): - with transaction.atomic(): - DumpUrl.objects.bulk_create( - [ - DumpUrl( - url=record["url"], - collection=collection, - scraped_text=record["full_text"], - scraped_title=record["title"], - ) - for record in batch - ] - ) - total_processed += len(batch) - print(f"Processed batch of {len(batch)} records. Total: {total_processed}") - - # Step 3: Check if classification is needed and queue if necessary - collection.queue_necessary_classifications() - - return f"Successfully processed {total_processed} records." - except Exception as e: - print(f"Error processing records: {str(e)}") - raise - - @celery_app.task() def migrate_dump_to_delta_and_handle_status_transistions(collection_id): """Task to migrate DumpUrls to DeltaUrls after classification is complete""" @@ -205,6 +103,8 @@ def migrate_dump_to_delta_and_handle_status_transistions(collection_id): initial_workflow_status = collection.workflow_status initial_reindexing_status = collection.reindexing_status + dump_count = collection.dump_urls.count() + curated_count = collection.curated_urls.count() # Migrate dump URLs to delta URLs collection.migrate_dump_to_delta() @@ -218,6 +118,7 @@ def migrate_dump_to_delta_and_handle_status_transistions(collection_id): WorkflowStatusChoices.READY_FOR_ENGINEERING, WorkflowStatusChoices.ENGINEERING_IN_PROGRESS, WorkflowStatusChoices.INDEXING_FINISHED_ON_DEV, + WorkflowStatusChoices.SCRAPING_SUCCESSFUL, ] if initial_workflow_status in pre_workflow_statuses: collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION @@ -228,4 +129,385 @@ def migrate_dump_to_delta_and_handle_status_transistions(collection_id): collection.reindexing_status = ReindexingStatusChoices.REINDEXING_READY_FOR_CURATION collection.save() + # Post the ingest summary for the scrape-ingest paths (WORKFLOW.md step 11). + # Sent from here, not the ingest task, because the delta counts only exist + # after migration. + if initial_workflow_status == WorkflowStatusChoices.SCRAPING_SUCCESSFUL or ( + initial_reindexing_status == ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV + ): + try: + send_detailed_import_notification( + collection_name=collection.name, + total_server_count=dump_count, + curated_count=curated_count, + dump_count=dump_count, + delta_count=collection.delta_urls.count(), + marked_for_deletion_count=collection.delta_urls.filter(to_delete=True).count(), + ) + except Exception as e: + logger.warning("Error sending ingest summary to Slack for %s: %s", collection.config_folder, e) + return f"Successfully migrated DumpUrls to DeltaUrls for collection {collection.name}." + + +@celery_app.task() +def dispatch_scrape_job(collection_id): + """Send a scrape job for the collection to the crawl4ai crawler via SSM. + + Triggered by READY_FOR_ENGINEERING and REINDEXING_NEEDED_ON_DEV (and manually via + the dispatch_scrape management command). On success records a ScrapeDispatch row — + the poller's freshness reference and the stall timeout's start time — and, on the + workflow path, moves READY_FOR_ENGINEERING to ENGINEERING_IN_PROGRESS so the board + shows the crawler holds the job. On failure sets SCRAPING_FAILED and records + nothing; the error never raises out of the task. + """ + Collection = apps.get_model("sde_collections", "Collection") + ScrapeDispatch = apps.get_model("sde_collections", "ScrapeDispatch") + collection = Collection.objects.get(id=collection_id) + + try: + command_id = send_job_to_crawler(collection) + except Exception as e: + logger.exception("Scrape dispatch failed for %s: %s", collection.config_folder, e) + _mark_scrape_failed(collection, f"dispatch error: {e}") + return None + + ScrapeDispatch.objects.create(collection=collection, ssm_command_id=command_id) + + # Compare-and-swap via .update(): only the workflow path advances (a re-scrape keeps + # its PROD_* status), and bypassing post_save means the dispatch trigger can't re-fire. + advanced = Collection.objects.filter( + id=collection_id, workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING + ).update(workflow_status=WorkflowStatusChoices.ENGINEERING_IN_PROGRESS) + if advanced: + _record_status_transition( + collection, WorkflowStatusChoices.READY_FOR_ENGINEERING, WorkflowStatusChoices.ENGINEERING_IN_PROGRESS + ) + return command_id + + +@celery_app.task() +def poll_scrape_jobs(): + """Beat task (every 5 min, gated on SCRAPE_POLL_ENABLED): find collections awaiting + crawl results and enqueue ingest for those whose run has completed. + + Scans READY_FOR_ENGINEERING and ENGINEERING_IN_PROGRESS (engineers flip to the latter + while a crawl runs — a collection must not strand there), plus the re-scrape path + (reindexing_status == REINDEXING_NEEDED_ON_DEV). Completion means a summary object + fresher than the latest ScrapeDispatch; with no fresh summary past the stall timeout + the run is declared dead. + """ + from datetime import timedelta + + from django.utils import timezone + + Collection = apps.get_model("sde_collections", "Collection") + + waiting = Collection.objects.filter( + workflow_status__in=[ + WorkflowStatusChoices.READY_FOR_ENGINEERING, + WorkflowStatusChoices.ENGINEERING_IN_PROGRESS, + ] + ) | Collection.objects.filter(reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV) + + stall_timeout = timedelta(hours=settings.SCRAPE_STALL_TIMEOUT_HOURS) + now = timezone.now() + enqueued = 0 + + for collection in waiting.distinct(): + dispatch = collection.scrape_dispatches.first() # Meta.ordering: latest first + if dispatch is None: + continue # never dispatched — nothing to poll against + + try: + summary = results_ready(collection.config_folder, dispatch.dispatched_at) + except Exception as e: + logger.error("Error checking S3 results for %s: %s", collection.config_folder, e) + continue + + if summary is not None: + ingest_scraped_collection.delay(collection.id) + enqueued += 1 + elif now - dispatch.dispatched_at > stall_timeout: + logger.error( + "Scrape stalled for %s: no fresh results %sh after dispatch", + collection.config_folder, + settings.SCRAPE_STALL_TIMEOUT_HOURS, + ) + _mark_scrape_failed(collection, f"no results {settings.SCRAPE_STALL_TIMEOUT_HOURS}h after dispatch") + + return f"Enqueued ingest for {enqueued} collection(s)." + + +PROD_STATUS_FOR_QC_STATUS = { + WorkflowStatusChoices.QUALITY_CHECK_PERFECT: WorkflowStatusChoices.PROD_PERFECT, + WorkflowStatusChoices.QUALITY_CHECK_MINOR: WorkflowStatusChoices.PROD_MINOR, +} + + +def _mint_run_id(): + import uuid + + from django.utils import timezone + + return f"{timezone.now().strftime('%Y-%m-%dT%H-%M-%SZ')}-{uuid.uuid4().hex[:6]}" + + +def _dispatch_index_run(collection_id, target, in_flight_status, failed_status): + """Shared body of the two indexing hand-off tasks: mint run_id -> export (manifest + last) -> RunTask -> record IndexDispatch -> set the in-flight status. Never indexes + in-process; failure lands on the target's failed status and never raises.""" + Collection = apps.get_model("sde_collections", "Collection") + collection = Collection.objects.get(id=collection_id) + run_id = _mint_run_id() + previous_status = collection.workflow_status + + try: + document_count = export_curated_to_s3(collection, target, run_id) + if document_count == 0: + raise ValueError("curated set is empty — nothing to index") + task_arn = run_index_task(collection, target, run_id) + except Exception as e: + logger.exception("Index dispatch (%s) failed for %s: %s", target, collection.config_folder, e) + collection.workflow_status = failed_status + collection.save() + return None + + IndexDispatch.objects.create( + collection=collection, + run_id=run_id, + target=target, + task_arn=task_arn, + previous_workflow_status=previous_status, + ) + collection.workflow_status = in_flight_status + collection.save() + logger.info( + "Dispatched %s index run %s for %s (%s documents)", target, run_id, collection.config_folder, document_count + ) + return run_id + + +@celery_app.task() +def index_collection_to_test(collection_id): + """Hand a curated collection to the WEB_COSMOS indexer, test target.""" + return _dispatch_index_run( + collection_id, + target=IndexDispatch.TARGET_TEST, + in_flight_status=WorkflowStatusChoices.TEST_INDEXING, + failed_status=WorkflowStatusChoices.INDEXING_FAILED_ON_TEST, + ) + + +@celery_app.task() +def index_collection_to_prod(collection_id): + """Hand a QC-passed collection to the WEB_COSMOS indexer, prod target.""" + return _dispatch_index_run( + collection_id, + target=IndexDispatch.TARGET_PROD, + in_flight_status=WorkflowStatusChoices.PRODUCTION_INDEXING, + failed_status=WorkflowStatusChoices.INDEXING_FAILED_ON_PROD, + ) + + +@celery_app.task() +def poll_index_runs(): + """Beat task (every 2 min, gated on INDEX_POLL_ENABLED): resolve in-flight index runs + by polling S3 status.json — never ecs.describe_tasks. + + Mapping: succeeded+test -> stay TEST_INDEXING and post the validation report to Slack + (curator sets QC from it); succeeded+prod -> PROD_PERFECT/PROD_MINOR mirroring the QC + status the run entered with; failed, unknown state, or stall timeout -> the target's + INDEXING_FAILED status. run_id namespacing means an old run's status.json can never + satisfy a newer dispatch. + """ + from datetime import timedelta + + from django.utils import timezone + + Collection = apps.get_model("sde_collections", "Collection") + + failed_status_for_target = { + IndexDispatch.TARGET_TEST: WorkflowStatusChoices.INDEXING_FAILED_ON_TEST, + IndexDispatch.TARGET_PROD: WorkflowStatusChoices.INDEXING_FAILED_ON_PROD, + } + stall_timeout = timedelta(hours=settings.INDEX_STALL_TIMEOUT_HOURS) + now = timezone.now() + resolved = 0 + + in_flight = Collection.objects.filter( + workflow_status__in=[ + WorkflowStatusChoices.TEST_INDEXING, + WorkflowStatusChoices.PRODUCTION_INDEXING, + ] + ) + for collection in in_flight: + dispatch = collection.index_dispatches.filter(completed_at__isnull=True).first() + if dispatch is None: + continue # nothing open to poll (e.g. status set by hand) + + try: + status = fetch_run_status(collection.config_folder, dispatch.run_id) + except Exception as e: + logger.error("Error checking index run for %s: %s", collection.config_folder, e) + continue + + if status is None: + if now - dispatch.dispatched_at > stall_timeout: + logger.error("Index run stalled for %s (run %s)", collection.config_folder, dispatch.run_id) + collection.workflow_status = failed_status_for_target[dispatch.target] + collection.save() + dispatch.completed_at = now + dispatch.save() + resolved += 1 + continue + + state = status.get("state") + if state == "succeeded": + if dispatch.target == IndexDispatch.TARGET_TEST: + # Collection stays at TEST_INDEXING; the curator sets QC from the report. + try: + validation = fetch_validation_report(collection.config_folder, dispatch.run_id) + send_indexing_validation_report(collection.name, dispatch.run_id, validation) + except Exception as e: + logger.warning("Error posting validation report for %s: %s", collection.config_folder, e) + else: + # Explicit map only: a prod run entered from anything other than a QC + # status (e.g. a manual re-dispatch) stays at PRODUCTION_INDEXING for a + # human to resolve rather than being silently promoted. + mirrored = PROD_STATUS_FOR_QC_STATUS.get(dispatch.previous_workflow_status) + if mirrored is None: + logger.warning( + "Prod index run %s for %s succeeded but entered from status %r; " + "leaving PRODUCTION_INDEXING for manual resolution", + dispatch.run_id, + collection.config_folder, + dispatch.previous_workflow_status, + ) + try: + send_slack_message( + f"Prod indexing of '{collection.name}' succeeded (run {dispatch.run_id}) but the run " + f"was not started from a QC status. Please set the final Prod status by hand." + ) + except Exception as e: + logger.warning("Error sending Slack message for %s: %s", collection.name, e) + else: + collection.workflow_status = mirrored + collection.save() + else: + # "failed" and any unknown state are both failures — needs_confirmation is + # reserved for future two-phase deletion and must not read as success. + logger.error( + "Index run for %s (run %s) ended state=%r error=%r", + collection.config_folder, + dispatch.run_id, + state, + status.get("error"), + ) + collection.workflow_status = failed_status_for_target[dispatch.target] + collection.save() + + dispatch.completed_at = now + dispatch.save() + resolved += 1 + + return f"Resolved {resolved} index run(s)." + + +@celery_app.task(soft_time_limit=600) +def ingest_scraped_collection(collection_id, claim=True): + """Ingest completed crawl results from S3 into DumpUrls (replaces fetch_full_text). + + The status transition IS the claim, executed as a compare-and-swap before any write: + ingest can outrun the 5-minute poll, and BaseUrl.url is globally unique, so two + concurrent ingests would die on IntegrityError mid-write. claim=False (manual + ingest_scrape_results command) skips the CAS — explicit operator intent — but the + delete-then-write body keeps the replay idempotent. + """ + Collection = apps.get_model("sde_collections", "Collection") + collection = Collection.objects.get(id=collection_id) + cid = collection.config_folder + + if claim: + dispatch = collection.scrape_dispatches.first() + summary = results_ready(cid, dispatch.dispatched_at if dispatch else None) + else: + found = fetch_summary(cid) + summary = found[0] if found else None + if summary is None: + return f"No (fresh) results for {cid}; nothing ingested." + + # Zero-document completion is a failure: without this, an empty crawl would + # "succeed" and silently publish an empty collection. + if summary.get("documents_scraped", 0) == 0: + _mark_scrape_failed(collection, "crawl completed with 0 documents") + return f"Scrape of {cid} completed with 0 documents; marked Scraping Failed." + + if claim: + old_status = collection.workflow_status + claimed = Collection.objects.filter( + id=collection_id, + workflow_status__in=[ + WorkflowStatusChoices.READY_FOR_ENGINEERING, + WorkflowStatusChoices.ENGINEERING_IN_PROGRESS, + ], + ).update(workflow_status=WorkflowStatusChoices.SCRAPING_SUCCESSFUL) + if claimed: + _record_status_transition(collection, old_status, WorkflowStatusChoices.SCRAPING_SUCCESSFUL) + else: + # Workflow path not claimable — try the re-scrape path. + claimed = Collection.objects.filter( + id=collection_id, + reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV, + ).update(reindexing_status=ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV) + if claimed == 0: + return f"{cid} already claimed by another ingest; exiting." + + try: + documents = _dedupe_by_url(fetch_documents(cid), cid) + with transaction.atomic(): + deleted_count, _ = DumpUrl.objects.filter(collection=collection).delete() + batch_size = 500 + for start in range(0, len(documents), batch_size): + DumpUrl.objects.bulk_create( + [ + DumpUrl( + url=document["url"], + collection=collection, + scraped_title=document.get("title") or "", + scraped_text=document.get("full_text") or "", + ) + for document in documents[start : start + batch_size] # noqa: E203 + ] + ) + logger.info("Ingested %s documents for %s (replaced %s).", len(documents), cid, deleted_count) + + collection.refresh_from_db() + collection.queue_necessary_classifications() + return f"Ingested {len(documents)} documents for {cid}." + except Exception as e: + # Never leave a claimed collection stuck in SCRAPING_SUCCESSFUL with no DumpUrls. + # On the re-scrape path the live workflow status is left alone (see _mark_scrape_failed). + logger.exception("Ingest failed for %s: %s", cid, e) + _mark_scrape_failed(collection, f"ingest error: {e}") + return None + + +def _dedupe_by_url(documents, cid): + """Drop repeated URLs within one crawl output (first occurrence wins). + + BaseUrl.url is unique, so a single duplicate would otherwise abort the whole + bulk_create inside the atomic block. + """ + seen = set() + unique = [] + for document in documents: + url = document.get("url") + if not url or url in seen: + continue + seen.add(url) + unique.append(document) + dropped = len(documents) - len(unique) + if dropped: + logger.warning("Dropped %s duplicate/blank-URL documents from crawl output for %s", dropped, cid) + return unique diff --git a/sde_collections/tests.py b/sde_collections/tests.py deleted file mode 100644 index daa70714..00000000 --- a/sde_collections/tests.py +++ /dev/null @@ -1,36 +0,0 @@ -from django.test import TestCase -from rest_framework.test import APIRequestFactory - -from .models.collection import Collection -from .tasks import import_candidate_urls_from_api, push_to_github_task - - -class CreateExcludePatternTestCase(TestCase): - def test_create_exclude_pattern(self): - factory = APIRequestFactory() - response = factory.post("/api/create-exclude-pattern", {"title": "new idea"}, format="json") - self.assertCountEqual(response, "hey") - - -class CreateIncludePatternTestCase(TestCase): - def test_create_include_pattern(self): - factory = APIRequestFactory() - response = factory.post("/api/create-include-pattern", {"title": "new idea"}, format="json") - self.assertCountEqual(response, "hey") - - -class ImportCandidateURLsTestCase(TestCase): - def test_import_all_candidate_urls_from_api(self): - import_candidate_urls_from_api() - self.assertEqual(1, 1) - - -class GitHubTestCase(TestCase): - fixtures = [ - "sde_collections/fixtures/collections.json", - ] - - def test_push_config_to_github(self): - collection = Collection.objects.first() - push_to_github_task([collection.id]) - self.assertEqual(collection.name, "NASA POWER") diff --git a/sde_collections/tests/test_aws_utils.py b/sde_collections/tests/test_aws_utils.py new file mode 100644 index 00000000..4638aa92 --- /dev/null +++ b/sde_collections/tests/test_aws_utils.py @@ -0,0 +1,74 @@ +# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_aws_utils.py + +from unittest.mock import patch + +import pytest +from django.test import override_settings + +from ..utils.aws import get_boto3_session + + +class TestGetBoto3Session: + @pytest.fixture(autouse=True) + def blank_pipeline_credentials(self): + """Pin the pipeline credentials to their base.py defaults: a developer's + .envs/.local may carry real SDE_AWS_* values, and these tests must not depend + on that.""" + with override_settings( + SDE_AWS_ACCESS_KEY_ID="", SDE_AWS_SECRET_ACCESS_KEY="", SDE_AWS_SESSION_TOKEN="", AWS_REGION="us-east-1" + ): + yield + + def test_default_chain_when_sde_keys_blank(self): + """With the SDE_AWS_* keys at their blank base.py defaults, the session must use the + default credential chain (instance role in AWS) — no explicit keys passed.""" + with patch("sde_collections.utils.aws.boto3.Session") as mock_session: + session = get_boto3_session() + + mock_session.assert_called_once_with(region_name="us-east-1") + assert session is mock_session.return_value + + @override_settings(SDE_AWS_ACCESS_KEY_ID="AKIATEST", SDE_AWS_SECRET_ACCESS_KEY="secret") + def test_explicit_keys_when_set(self): + with patch("sde_collections.utils.aws.boto3.Session") as mock_session: + session = get_boto3_session() + + mock_session.assert_called_once_with( + aws_access_key_id="AKIATEST", + aws_secret_access_key="secret", + aws_session_token=None, + region_name="us-east-1", + ) + assert session is mock_session.return_value + + @override_settings( + SDE_AWS_ACCESS_KEY_ID="ASIATEST", SDE_AWS_SECRET_ACCESS_KEY="secret", SDE_AWS_SESSION_TOKEN="tok" + ) + def test_session_token_passed_for_temporary_creds(self): + """SSO / STS credentials are a triple; the token must reach boto3 or every call is + rejected with InvalidClientTokenId.""" + with patch("sde_collections.utils.aws.boto3.Session") as mock_session: + get_boto3_session() + + mock_session.assert_called_once_with( + aws_access_key_id="ASIATEST", + aws_secret_access_key="secret", + aws_session_token="tok", + region_name="us-east-1", + ) + + @override_settings(SDE_AWS_ACCESS_KEY_ID="AKIATEST", SDE_AWS_SECRET_ACCESS_KEY="") + def test_partial_keys_fall_back_to_default_chain(self): + """One key without the other is a misconfiguration — fall back to the default chain + rather than passing an incomplete credential pair to boto3.""" + with patch("sde_collections.utils.aws.boto3.Session") as mock_session: + get_boto3_session() + + mock_session.assert_called_once_with(region_name="us-east-1") + + @override_settings(AWS_REGION="us-west-2") + def test_region_comes_from_settings(self): + with patch("sde_collections.utils.aws.boto3.Session") as mock_session: + get_boto3_session() + + mock_session.assert_called_once_with(region_name="us-west-2") diff --git a/sde_collections/tests/test_database_backup.py b/sde_collections/tests/test_database_backup.py index 680da2f1..5f895f96 100644 --- a/sde_collections/tests/test_database_backup.py +++ b/sde_collections/tests/test_database_backup.py @@ -3,7 +3,8 @@ import os import subprocess from datetime import datetime -from unittest.mock import Mock, patch +from io import StringIO +from unittest.mock import patch import pytest from django.core.management import call_command @@ -141,37 +142,56 @@ def test_handle_integration(self, compress, env_name, mock_subprocess, mock_date # Verify correct command execution mock_subprocess.assert_called_once() - # Verify correct filename used + # Verify the dump lands on the exact backups-volume path cmd_args = mock_subprocess.call_args[0][0] - date_str = "20240115" - expected_base = f"{env_name}_backup_{date_str}.sql" - assert cmd_args[-1].endswith(expected_base) + assert cmd_args[-1] == f"/app/backups/{env_name}_backup_20240115.sql" - # Verify cleanup attempted if compressed - if compress: - assert not os.path.exists(expected_base) + def test_compressed_backup_writes_gz_and_removes_temp_dump(self, mock_subprocess, tmp_path): + """Real file lifecycle: the mocked pg_dump writes a dump file, the real + compress_file gzips it, and temp_file_handler removes the intermediate .sql.""" + dump_content = b"-- PostgreSQL dump" - def test_handle_pg_dump_error(self, mock_subprocess, mock_date, monkeypatch): - """Test error handling when pg_dump fails.""" + def fake_pg_dump(cmd, env, check): + with open(cmd[-1], "wb") as f: + f.write(dump_content) + + mock_subprocess.side_effect = fake_pg_dump + output = tmp_path / "backup.sql" + + call_command("database_backup", output=str(output)) + + assert not output.exists() # temp dump cleaned up + with gzip.open(str(output) + ".gz", "rb") as f: + assert f.read() == dump_content + + def test_handle_pg_dump_error(self, mock_subprocess, tmp_path): + """pg_dump failure must be reported, swallowed, and leave no backup artifacts.""" mock_subprocess.side_effect = subprocess.CalledProcessError(1, "pg_dump") - monkeypatch.setenv("BACKUP_ENVIRONMENT", "staging") + output = tmp_path / "backup.sql" + out = StringIO() - call_command("database_backup") + call_command("database_backup", output=str(output), stdout=out) # must not raise - # Verify error handling and cleanup - date_str = "20240115" - temp_file = f"staging_backup_{date_str}.sql" - assert not os.path.exists(temp_file) + assert "Backup failed" in out.getvalue() + assert not output.exists() + assert not os.path.exists(str(output) + ".gz") - def test_handle_compression_error(self, mock_subprocess, mock_date, command, monkeypatch): - """Test error handling during compression.""" - monkeypatch.setenv("BACKUP_ENVIRONMENT", "staging") - # Mock compression to fail - command.compress_file = Mock(side_effect=Exception("Compression failed")) + def test_handle_compression_error(self, mock_subprocess, tmp_path): + """Compression failure must be reported, swallowed, and still clean up the temp + dump that pg_dump produced. (Patch compress_file on the class: call_command + instantiates its own Command, so patching a fixture instance guards nothing.)""" + + def fake_pg_dump(cmd, env, check): + with open(cmd[-1], "wb") as f: + f.write(b"-- dump") + + mock_subprocess.side_effect = fake_pg_dump + output = tmp_path / "backup.sql" + out = StringIO() - call_command("database_backup") + with patch.object(database_backup.Command, "compress_file", side_effect=Exception("Compression failed")): + call_command("database_backup", output=str(output), stdout=out) # must not raise - # Verify cleanup - date_str = "20240115" - temp_file = f"staging_backup_{date_str}.sql" - assert not os.path.exists(temp_file) + assert "Error during backup process" in out.getvalue() + assert not output.exists() # temp dump cleaned up despite the failure + assert not os.path.exists(str(output) + ".gz") diff --git a/sde_collections/tests/test_delta_patterns.py b/sde_collections/tests/test_delta_patterns.py index a7941fbd..0f85e61a 100644 --- a/sde_collections/tests/test_delta_patterns.py +++ b/sde_collections/tests/test_delta_patterns.py @@ -134,18 +134,17 @@ def test_apply_does_not_generate_delta_url_if_titles_match(self): generated_title=resolve_title(title_pattern, context), ) - # Create and apply a `DeltaTitlePattern` with the same title pattern - DeltaTitlePattern.objects.create( + # Creating the pattern auto-applies it (BaseMatchPattern.save calls apply) + pattern = DeltaTitlePattern.objects.create( collection=collection, match_pattern="https://example.com/*", match_pattern_type=2, title_pattern=title_pattern, ) - # pattern.apply() - - # Since the title matches, no new `DeltaUrl` should be created - DeltaUrl.objects.filter(url=curated_url.url).first() + # The pattern did match the CuratedUrl — so the absence of a DeltaUrl below + # is a deliberate skip on equal titles, not a failure to match. + assert pattern.get_matching_curated_urls().filter(url=curated_url.url).exists() assert not DeltaUrl.objects.filter(url=curated_url.url).exists() def test_apply_resolves_title_for_delta_urls(self): @@ -253,7 +252,7 @@ def test_pattern_reapplication_does_not_duplicate_delta_urls(self): ) delta_url.refresh_from_db() - delta_url.generated_title = "Title Before - Processed" + assert delta_url.generated_title == "Title Before - Processed" # applied on create # Promote to CuratedUrl collection.promote_to_curated() diff --git a/sde_collections/tests/test_import_fulltexts.py b/sde_collections/tests/test_import_fulltexts.py deleted file mode 100644 index 392c2ed4..00000000 --- a/sde_collections/tests/test_import_fulltexts.py +++ /dev/null @@ -1,114 +0,0 @@ -# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_import_fulltexts.py - -from unittest.mock import patch - -import pytest -from django.db.models.signals import post_save - -from inference.models.inference import ModelVersion -from inference.models.inference_choice_fields import ClassificationType -from sde_collections.models.collection import create_configs_on_status_change -from sde_collections.models.delta_url import DeltaUrl, DumpUrl -from sde_collections.tasks import ( - fetch_full_text, - migrate_dump_to_delta_and_handle_status_transistions, -) -from sde_collections.tests.factories import CollectionFactory - - -@pytest.fixture -def disconnect_signals(): - # Disconnect the signal before each test - post_save.disconnect(create_configs_on_status_change, sender="sde_collections.Collection") - yield - # Reconnect the signal after each test - post_save.connect(create_configs_on_status_change, sender="sde_collections.Collection") - - -@pytest.fixture -def model_version(): - """Create a model version for testing""" - return ModelVersion.objects.create( - api_identifier="test_model", - description="Test model version", - classification_type=ClassificationType.TDAMM, - is_active=True, - ) - - -@pytest.mark.django_db -def test_fetch_and_replace_full_text(disconnect_signals, model_version): - collection = CollectionFactory(config_folder="test_folder") - - mock_batch = [ - {"url": "http://example.com/1", "full_text": "Test Text 1", "title": "Test Title 1"}, - {"url": "http://example.com/2", "full_text": "Test Text 2", "title": "Test Title 2"}, - ] - - def mock_generator(): - yield (mock_batch) - - with patch("sde_collections.sinequa_api.Api.get_full_texts") as mock_get_full_texts, patch( - "sde_collections.sinequa_api.Api.get_total_count", return_value=2 - ), patch("sde_collections.utils.slack_utils.send_detailed_import_notification"): - mock_get_full_texts.return_value = mock_generator() - - # First fetch the full text - fetch_full_text(collection.id, "lrm_dev") - - # Verify DumpUrls were created - assert DumpUrl.objects.filter(collection=collection).count() == 2 - - # Then migrate the data - migrate_dump_to_delta_and_handle_status_transistions(collection.id) - - # Verify DeltaUrls were created - assert DeltaUrl.objects.filter(collection=collection).count() == 2 - - -@pytest.mark.django_db -def test_fetch_and_replace_full_text_large_dataset(disconnect_signals, model_version): - """Test processing a large number of records with proper pagination and batching.""" - collection = CollectionFactory(config_folder="test_folder") - - # Create sample data - 20,000 records in total - def create_batch(start_idx, size): - return [ - {"url": f"http://example.com/{i}", "full_text": f"Test Text {i}", "title": f"Test Title {i}"} - for i in range(start_idx, start_idx + size) - ] - - # Mock the API to return data in batches of 5000 (matching actual API pagination) - def mock_batch_generator(): - batch_size = 5000 - total_records = 20000 - - for start in range(0, total_records, batch_size): - yield (create_batch(start, min(batch_size, total_records - start))) - - with patch("sde_collections.sinequa_api.Api.get_full_texts") as mock_get_full_texts, patch( - "sde_collections.sinequa_api.Api.get_total_count", return_value=20000 - ), patch("sde_collections.utils.slack_utils.send_detailed_import_notification"): - mock_get_full_texts.return_value = mock_batch_generator() - - # Execute the fetch task - result = fetch_full_text(collection.id, "lrm_dev") - - # Verify DumpUrls were created - assert DumpUrl.objects.filter(collection=collection).count() == 20000 - - # Execute the migration task - migrate_result = migrate_dump_to_delta_and_handle_status_transistions(collection.id) - - # Verify total number of records - assert DeltaUrl.objects.filter(collection=collection).count() == 20000 - - # Verify some random records exist and have correct data - for i in [0, 4999, 5000, 19999]: # Check boundaries and middle - url = DeltaUrl.objects.get(url=f"http://example.com/{i}") - assert url.scraped_text == f"Test Text {i}" - assert url.scraped_title == f"Test Title {i}" - - # Verify batch processing worked by checking the success message - assert "Successfully processed 20000 records" in result - assert "Successfully migrated DumpUrls to DeltaUrls" in migrate_result diff --git a/sde_collections/tests/test_indexing_dispatch.py b/sde_collections/tests/test_indexing_dispatch.py new file mode 100644 index 00000000..30b6c6cb --- /dev/null +++ b/sde_collections/tests/test_indexing_dispatch.py @@ -0,0 +1,479 @@ +# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_indexing_dispatch.py + +import io +import json +from datetime import timedelta +from unittest.mock import MagicMock, patch + +import pytest +from botocore.exceptions import ClientError +from django.test import override_settings +from django.utils import timezone + +from sde_collections.indexing.dispatch import run_index_task +from sde_collections.indexing.export import export_curated_to_s3 +from sde_collections.indexing.run_status import ( + fetch_run_status, + fetch_validation_report, +) +from sde_collections.models.collection_choice_fields import ( + Divisions, + DocumentTypes, + WorkflowStatusChoices, +) +from sde_collections.models.delta_patterns import DeltaExcludePattern +from sde_collections.models.indexing import IndexDispatch +from sde_collections.tasks import ( + index_collection_to_prod, + index_collection_to_test, + poll_index_runs, +) +from sde_collections.tests.factories import CollectionFactory, CuratedUrlFactory + +RUN_ID = "2026-08-13T20-00-00Z-abc123" + +INDEXING_SETTINGS = dict( + SDE_INDEX_BUCKET="sde-cosmos-indexing-dev", + INDEXING_ECS_CLUSTER="api-scrapers-cluster-dev", + INDEXING_TASK_FAMILY="web_cosmos-scraper-dev", + INDEXING_DISPATCH_ROLE_ARN="arn:aws:iam::123456789012:role/CosmosIndexingDispatchRole-dev", +) + + +class ExportCapture: + """Mock S3 client capturing the export write sequence and payloads.""" + + def __init__(self): + self.calls = [] # ordered (kind, key) + self.jsonl_lines = None + self.manifest = None + self.client = MagicMock() + self.client.upload_fileobj.side_effect = self._upload_fileobj + self.client.put_object.side_effect = self._put_object + + def _upload_fileobj(self, fileobj, bucket, key): + content = fileobj.read().decode("utf-8") + self.jsonl_lines = [json.loads(line) for line in content.splitlines() if line] + self.calls.append(("documents", key)) + + def _put_object(self, Bucket, Key, Body, **kwargs): + self.manifest = json.loads(Body) + self.calls.append(("manifest", Key)) + + +@pytest.fixture +def s3_export(): + capture = ExportCapture() + session = MagicMock() + session.client.return_value = capture.client + with patch("sde_collections.indexing.export.get_boto3_session", return_value=session): + yield capture + + +@pytest.mark.django_db +class TestExportCuratedToS3: + @pytest.fixture(autouse=True) + def indexing_settings(self): + with override_settings(**INDEXING_SETTINGS): + yield + + def test_manifest_written_last_with_exact_count_and_keys(self, s3_export): + collection = CollectionFactory() + CuratedUrlFactory.create_batch(3, collection=collection) + + count = export_curated_to_s3(collection, "test", RUN_ID) + + assert count == 3 + assert [kind for kind, _ in s3_export.calls] == ["documents", "manifest"] + prefix = f"curated_collections/{collection.config_folder}/{RUN_ID}" + assert s3_export.calls[0][1] == f"{prefix}/documents.jsonl" + assert s3_export.calls[1][1] == f"{prefix}/manifest.json" + assert s3_export.manifest["document_count"] == 3 + assert s3_export.manifest["collection_key"] == collection.config_folder + assert s3_export.manifest["run_id"] == RUN_ID + assert s3_export.manifest["target"] == "test" + assert s3_export.manifest["schema_version"] == 1 + + def test_excluded_urls_are_not_exported(self, s3_export): + collection = CollectionFactory() + CuratedUrlFactory(collection=collection, url="https://keep.nasa.gov/a") + excluded_url = CuratedUrlFactory(collection=collection, url="https://exclude.nasa.gov/b") + pattern = DeltaExcludePattern.objects.create(collection=collection, match_pattern="https://exclude.nasa.gov/b") + # The excluded annotation reads this M2M; population is the pattern system's job + # (covered in test_promote_collection) — here we only care that export honours it. + pattern.curated_urls.add(excluded_url) + + count = export_curated_to_s3(collection, "test", RUN_ID) + + assert count == 1 + assert s3_export.manifest["document_count"] == 1 + assert [line["url"] for line in s3_export.jsonl_lines] == ["https://keep.nasa.gov/a"] + + def test_title_and_label_resolution(self, s3_export): + collection = CollectionFactory(division=Divisions.ASTROPHYSICS, document_type=DocumentTypes.DATA) + CuratedUrlFactory( + collection=collection, + generated_title="Generated wins", + scraped_title="Scraped loses", + document_type=DocumentTypes.IMAGES, # differs from collection default -> exported + division=Divisions.ASTROPHYSICS, # equals collection default -> omitted + ) + + export_curated_to_s3(collection, "test", RUN_ID) + + line = s3_export.jsonl_lines[0] + assert line["title"] == "Generated wins" + assert line["document_type"] == "Images" # label, not the int + assert "division" not in line + assert line["is_metadata_viewer"] is False + assert s3_export.manifest["division"] == "Astrophysics" + assert s3_export.manifest["document_type"] == "Data" + + @override_settings(SDE_INDEX_BUCKET="") + def test_blank_bucket_refuses_to_export(self, s3_export): + collection = CollectionFactory() + + with pytest.raises(ValueError, match="SDE_INDEX_BUCKET"): + export_curated_to_s3(collection, "test", RUN_ID) + + +@pytest.mark.django_db +class TestRunIndexTask: + @override_settings(INDEXING_DISPATCH_ROLE_ARN="", INDEXING_ECS_CLUSTER="", INDEXING_TASK_FAMILY="") + def test_unconfigured_settings_refuse_to_dispatch(self): + """Dev-only guard: with the blank defaults, no dispatch can leave the box.""" + collection = CollectionFactory() + + with pytest.raises(ValueError, match="INDEXING_ECS_CLUSTER"): + run_index_task(collection, "test", RUN_ID) + + @override_settings(**{**INDEXING_SETTINGS, "INDEXING_DISPATCH_ROLE_ARN": ""}) + @patch("sde_collections.indexing.dispatch.boto3.client") + @patch("sde_collections.indexing.dispatch.get_boto3_session") + def test_blank_role_arn_runs_task_with_session_creds(self, mock_session, mock_boto3_client): + """Local dev: no role to assume, so RunTask goes straight through the pipeline + session (never through a raw boto3.client with assumed creds).""" + collection = CollectionFactory() + ecs = MagicMock() + ecs.run_task.return_value = {"tasks": [{"taskArn": "arn:aws:ecs:task/direct"}], "failures": []} + mock_session.return_value.client.return_value = ecs + + arn = run_index_task(collection, "test", RUN_ID) + + assert arn == "arn:aws:ecs:task/direct" + mock_session.return_value.client.assert_called_once_with("ecs") + mock_boto3_client.assert_not_called() + assert ecs.run_task.call_args.kwargs["taskDefinition"] == "web_cosmos-scraper-dev" + + @override_settings(**INDEXING_SETTINGS) + @patch("sde_collections.indexing.dispatch.boto3.client") + @patch("sde_collections.indexing.dispatch.get_boto3_session") + def test_dispatch_assumes_role_and_runs_task(self, mock_session, mock_boto3_client): + collection = CollectionFactory() + sts = MagicMock() + sts.assume_role.return_value = { + "Credentials": {"AccessKeyId": "AK", "SecretAccessKey": "SK", "SessionToken": "TOK"} + } + mock_session.return_value.client.return_value = sts + ecs = MagicMock() + ecs.run_task.return_value = {"tasks": [{"taskArn": "arn:aws:ecs:task/1"}], "failures": []} + mock_boto3_client.return_value = ecs + + arn = run_index_task(collection, "test", RUN_ID) + + assert arn == "arn:aws:ecs:task/1" + assert sts.assume_role.call_args.kwargs["RoleArn"] == INDEXING_SETTINGS["INDEXING_DISPATCH_ROLE_ARN"] + run_kwargs = ecs.run_task.call_args.kwargs + assert run_kwargs["cluster"] == "api-scrapers-cluster-dev" + assert run_kwargs["taskDefinition"] == "web_cosmos-scraper-dev" + command = run_kwargs["overrides"]["containerOverrides"][0]["command"] + # The executable must lead the list: a container override replaces the task + # definition's command wholesale, and the indexer image has no ENTRYPOINT. + assert command == [ + "python3", + "api_scraper.py", + "--source", + "WEB_COSMOS", + "--collection", + collection.config_folder, + "--target", + "test", + "--run-id", + RUN_ID, + ] + + def _run_with_network(self, subnets, security_groups, assign_public_ip=True): + collection = CollectionFactory() + with ( + override_settings( + **INDEXING_SETTINGS, + INDEXING_SUBNETS=subnets, + INDEXING_SECURITY_GROUPS=security_groups, + INDEXING_ASSIGN_PUBLIC_IP=assign_public_ip, + ), + patch("sde_collections.indexing.dispatch.boto3.client") as mock_boto3_client, + patch("sde_collections.indexing.dispatch.get_boto3_session") as mock_session, + ): + mock_session.return_value.client.return_value.assume_role.return_value = { + "Credentials": {"AccessKeyId": "AK", "SecretAccessKey": "SK", "SessionToken": "TOK"} + } + ecs = mock_boto3_client.return_value + ecs.run_task.return_value = {"tasks": [{"taskArn": "arn:aws:ecs:task/1"}], "failures": []} + run_index_task(collection, "test", RUN_ID) + return ecs.run_task.call_args.kwargs + + def test_network_config_is_built_from_settings(self): + kwargs = self._run_with_network("subnet-a,subnet-b", "sg-1", assign_public_ip=False) + + assert kwargs["networkConfiguration"]["awsvpcConfiguration"] == { + "subnets": ["subnet-a", "subnet-b"], + "securityGroups": ["sg-1"], + "assignPublicIp": "DISABLED", + } + + def test_no_network_config_when_both_blank(self): + assert "networkConfiguration" not in self._run_with_network("", "") + + def test_subnets_without_security_groups_refuse_to_dispatch(self): + """Half-configured networking must fail loudly, not send an empty securityGroups list.""" + with pytest.raises(ValueError, match="INDEXING_SUBNETS and INDEXING_SECURITY_GROUPS"): + self._run_with_network("subnet-a", "") + + @override_settings(**INDEXING_SETTINGS) + @patch("sde_collections.indexing.dispatch.boto3.client") + @patch("sde_collections.indexing.dispatch.get_boto3_session") + def test_runtask_failures_raise(self, mock_session, mock_boto3_client): + collection = CollectionFactory() + mock_session.return_value.client.return_value.assume_role.return_value = { + "Credentials": {"AccessKeyId": "AK", "SecretAccessKey": "SK", "SessionToken": "TOK"} + } + mock_boto3_client.return_value.run_task.return_value = { + "tasks": [], + "failures": [{"reason": "MISSING"}], + } + + with pytest.raises(RuntimeError, match="failures"): + run_index_task(collection, "test", RUN_ID) + + +@pytest.mark.django_db +class TestIndexDispatchTasks: + @patch("sde_collections.tasks.run_index_task", return_value="arn:aws:ecs:task/1") + @patch("sde_collections.tasks.export_curated_to_s3", return_value=25) + def test_test_dispatch_records_and_sets_status(self, mock_export, mock_run): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.CURATED) + + run_id = index_collection_to_test(collection.id) + + dispatch = IndexDispatch.objects.get(collection=collection) + assert dispatch.run_id == run_id + assert dispatch.target == "test" + assert dispatch.task_arn == "arn:aws:ecs:task/1" + assert dispatch.previous_workflow_status == WorkflowStatusChoices.CURATED + assert dispatch.completed_at is None + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.TEST_INDEXING + # export happens before dispatch, with the same run_id + assert mock_export.call_args.args[1] == "test" + assert mock_export.call_args.args[2] == run_id + + @patch("sde_collections.tasks.run_index_task") + @patch("sde_collections.tasks.export_curated_to_s3", side_effect=Exception("S3 down")) + def test_export_failure_lands_on_failed_status_without_raising(self, mock_export, mock_run): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.CURATED) + + assert index_collection_to_test(collection.id) is None + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.INDEXING_FAILED_ON_TEST + assert not IndexDispatch.objects.filter(collection=collection).exists() + mock_run.assert_not_called() + + @patch("sde_collections.tasks.run_index_task") + @patch("sde_collections.tasks.export_curated_to_s3", return_value=0) + def test_empty_curated_set_fails_before_dispatch(self, mock_export, mock_run): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.CURATED) + + index_collection_to_test(collection.id) + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.INDEXING_FAILED_ON_TEST + mock_run.assert_not_called() + + @patch("sde_collections.tasks.run_index_task", return_value="arn:aws:ecs:task/2") + @patch("sde_collections.tasks.export_curated_to_s3", return_value=10) + def test_prod_dispatch_remembers_entering_qc_status(self, mock_export, mock_run): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.QUALITY_CHECK_MINOR) + + index_collection_to_prod(collection.id) + + dispatch = IndexDispatch.objects.get(collection=collection) + assert dispatch.target == "prod" + assert dispatch.previous_workflow_status == WorkflowStatusChoices.QUALITY_CHECK_MINOR + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.PRODUCTION_INDEXING + + +class TestRunStatusFetchers: + """The S3 boundary itself: poll_index_runs tests mock fetch_run_status, so the real + index_runs/{config_folder}/{run_id}/ key layout and missing-key handling (None while + the run is in flight, raise on real errors) must be proven here.""" + + @pytest.fixture(autouse=True) + def index_bucket(self): + # override_settings can't decorate a plain (non-SimpleTestCase) class + with override_settings(SDE_INDEX_BUCKET="sde-cosmos-indexing-test"): + yield + + def _patch_s3(self, get_object): + s3 = MagicMock() + if isinstance(get_object, BaseException): + s3.get_object.side_effect = get_object + else: + s3.get_object.return_value = get_object + session = MagicMock() + session.client.return_value = s3 + return patch("sde_collections.indexing.run_status.get_boto3_session", return_value=session), s3 + + def test_fetch_run_status_reads_status_json_under_run_prefix(self): + payload = {"state": "succeeded"} + patcher, s3 = self._patch_s3({"Body": io.BytesIO(json.dumps(payload).encode("utf-8"))}) + + with patcher: + assert fetch_run_status("astro_data", RUN_ID) == payload + + s3.get_object.assert_called_once_with( + Bucket="sde-cosmos-indexing-test", Key=f"index_runs/astro_data/{RUN_ID}/status.json" + ) + + def test_fetch_validation_report_reads_validation_json(self): + payload = {"count_matches": True} + patcher, s3 = self._patch_s3({"Body": io.BytesIO(json.dumps(payload).encode("utf-8"))}) + + with patcher: + assert fetch_validation_report("astro_data", RUN_ID) == payload + + s3.get_object.assert_called_once_with( + Bucket="sde-cosmos-indexing-test", Key=f"index_runs/astro_data/{RUN_ID}/validation.json" + ) + + @pytest.mark.parametrize("code", ["NoSuchKey", "404"]) + def test_missing_status_means_run_in_flight(self, code): + patcher, _ = self._patch_s3(ClientError({"Error": {"Code": code}}, "GetObject")) + + with patcher: + assert fetch_run_status("astro_data", RUN_ID) is None + + def test_real_s3_errors_propagate(self): + """AccessDenied etc. must surface, not read as 'still indexing' until stall timeout.""" + patcher, _ = self._patch_s3(ClientError({"Error": {"Code": "AccessDenied"}}, "GetObject")) + + with patcher: + with pytest.raises(ClientError): + fetch_run_status("astro_data", RUN_ID) + + +def _dispatched(collection, target, previous, run_id="run-1"): + return IndexDispatch.objects.create( + collection=collection, + run_id=run_id, + target=target, + task_arn="arn:aws:ecs:task/x", + previous_workflow_status=previous, + ) + + +@pytest.mark.django_db +class TestPollIndexRuns: + @patch("sde_collections.tasks.send_indexing_validation_report") + @patch("sde_collections.tasks.fetch_validation_report", return_value={"count_matches": True}) + @patch("sde_collections.tasks.fetch_run_status", return_value={"state": "succeeded"}) + def test_test_success_stays_and_posts_validation_once(self, mock_status, mock_validation, mock_slack): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.TEST_INDEXING) + dispatch = _dispatched(collection, "test", WorkflowStatusChoices.CURATED) + + poll_index_runs() + poll_index_runs() # resolved dispatches are never polled again + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.TEST_INDEXING + mock_slack.assert_called_once_with(collection.name, dispatch.run_id, {"count_matches": True}) + dispatch.refresh_from_db() + assert dispatch.completed_at is not None + + @pytest.mark.parametrize( + "entered_with,expected", + [ + (WorkflowStatusChoices.QUALITY_CHECK_PERFECT, WorkflowStatusChoices.PROD_PERFECT), + (WorkflowStatusChoices.QUALITY_CHECK_MINOR, WorkflowStatusChoices.PROD_MINOR), + ], + ) + @patch("sde_collections.tasks.fetch_run_status", return_value={"state": "succeeded"}) + def test_prod_success_mirrors_entering_qc_status(self, mock_status, entered_with, expected): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.PRODUCTION_INDEXING) + _dispatched(collection, "prod", entered_with) + + poll_index_runs() + + collection.refresh_from_db() + assert collection.workflow_status == expected + + def test_prod_success_from_non_qc_status_holds_for_manual_resolution(self): + """The mirror map is explicit: a prod run that did not enter from a QC status + (e.g. a manual re-dispatch) is never silently promoted to PROD_PERFECT.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.PRODUCTION_INDEXING) + dispatch = _dispatched(collection, "prod", WorkflowStatusChoices.INDEXING_FAILED_ON_PROD) + + with ( + patch("sde_collections.tasks.fetch_run_status", return_value={"state": "succeeded"}), + patch("sde_collections.tasks.send_slack_message") as mock_slack, + ): + poll_index_runs() + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.PRODUCTION_INDEXING + dispatch.refresh_from_db() + assert dispatch.completed_at is not None # resolved: not re-polled + mock_slack.assert_called_once() + assert "by hand" in mock_slack.call_args.args[0] + + @pytest.mark.parametrize("state", ["failed", "needs_confirmation", "something_new"]) + @patch("sde_collections.tasks.fetch_run_status") + def test_failed_and_unknown_states_land_on_failed_status(self, mock_status, state): + mock_status.return_value = {"state": state, "error": "deletion_threshold_exceeded"} + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.TEST_INDEXING) + _dispatched(collection, "test", WorkflowStatusChoices.CURATED) + + poll_index_runs() + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.INDEXING_FAILED_ON_TEST + + @patch("sde_collections.tasks.fetch_run_status", return_value=None) + def test_in_flight_run_stays_open_until_stall_timeout(self, mock_status): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.PRODUCTION_INDEXING) + dispatch = _dispatched(collection, "prod", WorkflowStatusChoices.QUALITY_CHECK_PERFECT) + + poll_index_runs() + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.PRODUCTION_INDEXING + + IndexDispatch.objects.filter(id=dispatch.id).update( + dispatched_at=timezone.now() - timedelta(hours=7) # default timeout is 6h + ) + poll_index_runs() + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.INDEXING_FAILED_ON_PROD + + @patch("sde_collections.tasks.fetch_run_status", return_value=None) + def test_poller_reads_only_the_open_dispatchs_run_id(self, mock_status): + """run_id namespacing: an old (resolved) run's status.json can never satisfy a + newer dispatch — the poller only ever queries the open dispatch's run_id.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.TEST_INDEXING) + old = _dispatched(collection, "test", WorkflowStatusChoices.CURATED, run_id="run-old") + IndexDispatch.objects.filter(id=old.id).update(completed_at=timezone.now()) + _dispatched(collection, "test", WorkflowStatusChoices.CURATED, run_id="run-new") + + poll_index_runs() + + assert mock_status.call_args.args == (collection.config_folder, "run-new") diff --git a/sde_collections/tests/test_inference_flag.py b/sde_collections/tests/test_inference_flag.py new file mode 100644 index 00000000..0c0c4ed1 --- /dev/null +++ b/sde_collections/tests/test_inference_flag.py @@ -0,0 +1,105 @@ +# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_inference_flag.py + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from django.test import override_settings + +from inference.models import InferenceJob +from inference.signals import create_periodic_tasks +from inference.tasks import process_inference_job_queue +from sde_collections.models.collection import Collection +from sde_collections.tests.factories import CollectionFactory + +TDAMM_FOLDER = "imagine_the_universe" + +# These tests call the REAL queue_necessary_classifications (patching only the migration +# task's delay) — patching the whole method, as the older trigger tests do, would guard +# nothing about the flag behaviour. + + +def _collection_with_folder(config_folder): + collection = CollectionFactory() + Collection.objects.filter(id=collection.id).update(config_folder=config_folder) + collection.refresh_from_db() + return collection + + +@pytest.mark.django_db +@override_settings(INFERENCE_ENABLED=False) +@patch("sde_collections.models.collection.migrate_dump_to_delta_and_handle_status_transistions.delay") +def test_tdamm_collection_migrates_directly_when_inference_disabled(mock_migrate): + """Regression guard: with the flag off, a TDAMM-listed collection must NOT get an + InferenceJob (it would queue forever and never reach Ready for Curation).""" + collection = _collection_with_folder(TDAMM_FOLDER) + + collection.queue_necessary_classifications() + + mock_migrate.assert_called_once_with(collection.id) + assert not InferenceJob.objects.filter(collection=collection).exists() + + +@pytest.mark.django_db +@override_settings(INFERENCE_ENABLED=False) +@patch("sde_collections.models.collection.migrate_dump_to_delta_and_handle_status_transistions.delay") +def test_regular_collection_migrates_directly_when_inference_disabled(mock_migrate): + collection = _collection_with_folder("some_regular_collection") + + collection.queue_necessary_classifications() + + mock_migrate.assert_called_once_with(collection.id) + assert not InferenceJob.objects.filter(collection=collection).exists() + + +@pytest.mark.django_db +@override_settings(INFERENCE_ENABLED=True) +@patch("sde_collections.models.collection.migrate_dump_to_delta_and_handle_status_transistions.delay") +def test_tdamm_collection_creates_inference_job_when_enabled(mock_migrate): + collection = _collection_with_folder(TDAMM_FOLDER) + + collection.queue_necessary_classifications() + + assert InferenceJob.objects.filter(collection=collection).exists() + mock_migrate.assert_not_called() + + +def _run_post_migrate_handler(): + create_periodic_tasks(sender=SimpleNamespace(name="inference")) + + +@pytest.mark.django_db +@pytest.mark.parametrize("flag", [False, True]) +def test_periodic_task_rows_follow_the_flag(flag): + from django_celery_beat.models import PeriodicTask + + with override_settings(INFERENCE_ENABLED=flag): + _run_post_migrate_handler() + + rows = PeriodicTask.objects.filter(task="inference.tasks.process_inference_job_queue") + assert rows.count() == 2 + assert all(row.enabled is flag for row in rows) + + +@pytest.mark.django_db +@override_settings(INFERENCE_ENABLED=False) +def test_periodic_task_disable_survives_re_migrate(): + """A hand-enable in the admin must not survive a deploy's migrate step: the signal + re-asserts enabled from the flag on every post_migrate.""" + from django_celery_beat.models import PeriodicTask + + _run_post_migrate_handler() + PeriodicTask.objects.filter(task="inference.tasks.process_inference_job_queue").update(enabled=True) + + _run_post_migrate_handler() + + rows = PeriodicTask.objects.filter(task="inference.tasks.process_inference_job_queue") + assert rows.count() == 2 + assert all(row.enabled is False for row in rows) + + +@override_settings(INFERENCE_ENABLED=False) +def test_process_inference_job_queue_short_circuits_when_disabled(): + """Belt and braces: even a manually-triggered run must refuse to process the queue. + (No django_db marker: the early return must not touch the database at all.)""" + assert process_inference_job_queue() == "Inference pipeline disabled (INFERENCE_ENABLED=False)" diff --git a/sde_collections/tests/test_management_commands.py b/sde_collections/tests/test_management_commands.py new file mode 100644 index 00000000..8c2988c3 --- /dev/null +++ b/sde_collections/tests/test_management_commands.py @@ -0,0 +1,65 @@ +# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_management_commands.py + +from io import StringIO +from unittest.mock import patch + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError + +from sde_collections.tests.factories import CollectionFactory + + +@pytest.mark.django_db +class TestDispatchScrapeCommand: + def test_unknown_collection_is_a_command_error(self): + with pytest.raises(CommandError, match="No collection"): + call_command("dispatch_scrape", collection="does_not_exist") + + @patch("sde_collections.management.commands.dispatch_scrape.dispatch_scrape_job", return_value="cmd-1") + def test_runs_the_task_synchronously_and_reports_command_id(self, mock_task): + collection = CollectionFactory() + out = StringIO() + + call_command("dispatch_scrape", collection=collection.config_folder, stdout=out) + + mock_task.assert_called_once_with(collection.id) # called directly, not .delay + assert "cmd-1" in out.getvalue() + + @patch("sde_collections.management.commands.dispatch_scrape.dispatch_scrape_job", return_value=None) + def test_dispatch_failure_is_a_command_error(self, mock_task): + collection = CollectionFactory() + + with pytest.raises(CommandError, match="Dispatch failed"): + call_command("dispatch_scrape", collection=collection.config_folder) + + +@pytest.mark.django_db +class TestIngestScrapeResultsCommand: + def test_unknown_collection_is_a_command_error(self): + with pytest.raises(CommandError, match="No collection"): + call_command("ingest_scrape_results", collection="does_not_exist") + + @patch( + "sde_collections.management.commands.ingest_scrape_results.ingest_scraped_collection", + return_value="Ingested 3 documents", + ) + def test_manual_ingest_skips_the_claim(self, mock_task): + """The command's contract: explicit operator intent bypasses the status CAS.""" + collection = CollectionFactory() + out = StringIO() + + call_command("ingest_scrape_results", collection=collection.config_folder, stdout=out) + + mock_task.assert_called_once_with(collection.id, claim=False) + assert "Ingested 3 documents" in out.getvalue() + + @patch( + "sde_collections.management.commands.ingest_scrape_results.ingest_scraped_collection", + return_value=None, + ) + def test_ingest_failure_is_a_command_error(self, mock_task): + collection = CollectionFactory() + + with pytest.raises(CommandError, match="Ingest failed"): + call_command("ingest_scrape_results", collection=collection.config_folder) diff --git a/sde_collections/tests/test_migrate_dump.py b/sde_collections/tests/test_migrate_dump.py index a1a3bd12..31d3360a 100644 --- a/sde_collections/tests/test_migrate_dump.py +++ b/sde_collections/tests/test_migrate_dump.py @@ -3,6 +3,8 @@ import pytest +# The production list — a local copy would silently drift and stop covering new fields. +from sde_collections.models.collection import DELTA_COMPARISON_FIELDS # noqa: E402 from sde_collections.models.collection_choice_fields import Divisions, DocumentTypes from sde_collections.models.delta_patterns import ( DeltaDocumentTypePattern, @@ -16,8 +18,6 @@ DumpUrlFactory, ) -DELTA_COMPARISON_FIELDS = ["scraped_title", "tdamm_tag", "division"] # Assuming a central definition - @pytest.mark.django_db class TestMigrationHelpers: @@ -84,7 +84,12 @@ def test_identical_url_in_both(self): collection = CollectionFactory() # Create DumpUrl with specific values - dump_url = DumpUrlFactory(collection=collection, scraped_title="Same Title", division=Divisions.ASTROPHYSICS) + dump_url = DumpUrlFactory( + collection=collection, + scraped_title="Same Title", + scraped_text="Same text", + division=Divisions.ASTROPHYSICS, + ) # Ensure tdamm_tag is explicitly set to match dump_url.tdamm_tag_manual = [] @@ -96,6 +101,7 @@ def test_identical_url_in_both(self): collection=collection, url=dump_url.url, # Use the same URL scraped_title="Same Title", + scraped_text="Same text", division=Divisions.ASTROPHYSICS, ) @@ -230,20 +236,19 @@ def test_full_migration_deleted_url(self): @pytest.mark.django_db def test_empty_delta_comparison_fields(): - collection = CollectionFactory() - dump_url = DumpUrlFactory(collection=collection, scraped_title="Same Title") - CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Same Title") # noqa + """The comparison-field list is what drives delta creation: with it patched empty + in the PRODUCTION module, even a differing title must not produce a DeltaUrl. + (Rebinding this test module's import would be a no-op — patch where it's used.)""" + from unittest.mock import patch - global DELTA_COMPARISON_FIELDS - original_fields = DELTA_COMPARISON_FIELDS - DELTA_COMPARISON_FIELDS = [] # Simulate empty comparison fields + collection = CollectionFactory() + dump_url = DumpUrlFactory(collection=collection, scraped_title="New Title") + CuratedUrlFactory(collection=collection, url=dump_url.url, scraped_title="Old Title") - try: + with patch("sde_collections.models.collection.DELTA_COMPARISON_FIELDS", []): collection.migrate_dump_to_delta() - # No DeltaUrl should be created as there are no fields to compare - assert not DeltaUrl.objects.filter(url=dump_url.url).exists() - finally: - DELTA_COMPARISON_FIELDS = original_fields # Reset the fields after test + + assert not DeltaUrl.objects.filter(url=dump_url.url).exists() @pytest.mark.django_db diff --git a/sde_collections/tests/test_promote_collection.py b/sde_collections/tests/test_promote_collection.py index 8791efae..eb01d98c 100644 --- a/sde_collections/tests/test_promote_collection.py +++ b/sde_collections/tests/test_promote_collection.py @@ -138,12 +138,14 @@ def test_promotion_with_overlapping_patterns_and_deletion(): for url in urls: DeltaUrl.objects.create(collection=collection, url=url, scraped_title=f"Title for {url}") - # Create overlapping patterns that will affect the same URLs + # Create overlapping patterns that will affect the same URLs. + # NOTE: match patterns are *-wildcards (get_regex_pattern re.escape()s everything + # else) — regex-idiom patterns like ".*docs.*" silently match nothing. patterns = [ - {"pattern": ".*docs.*", "title": "Documentation: {title}"}, - {"pattern": ".*guide.*", "title": "Guide: {title}"}, - {"pattern": ".*api.*", "title": "API: {title}"}, - {"pattern": ".*doc[0-9]", "title": "Doc Number: {title}"}, + {"pattern": "*docs*", "title": "Documentation: {title}"}, + {"pattern": "*guide*", "title": "Guide: {title}"}, + {"pattern": "*api*", "title": "API: {title}"}, + {"pattern": "*api/v1/doc*", "title": "Doc Number: {title}"}, ] # Create and apply multiple patterns @@ -161,27 +163,31 @@ def test_promotion_with_overlapping_patterns_and_deletion(): # Initial promotion collection.promote_to_curated() - # Verify our complex setup + # Every pattern's curated_urls relation must hold exactly its wildcard matches + expected_matches = { + "*docs*": {"https://example.com/docs/guide1", "https://example.com/docs/guide2"}, + "*guide*": {"https://example.com/docs/guide1", "https://example.com/docs/guide2"}, + "*api*": {"https://example.com/api/v1/doc1", "https://example.com/api/v1/doc2"}, + "*api/v1/doc*": {"https://example.com/api/v1/doc1", "https://example.com/api/v1/doc2"}, + } + assert CuratedUrl.objects.filter(collection=collection).count() == 4 for pattern in title_patterns: - matching_urls = pattern.curated_urls.all() - print(f"\nPattern '{pattern.match_pattern}' matches {matching_urls.count()} URLs:") - for url in matching_urls: - print(f"- {url.url}") + matched = set(pattern.curated_urls.values_list("url", flat=True)) + assert matched == expected_matches[pattern.match_pattern], pattern.match_pattern # Now create deletion DeltaUrls but with overlapping pattern matches urls_to_delete = ["https://example.com/docs/guide1", "https://example.com/api/v1/doc1"] for url in urls_to_delete: DeltaUrl.objects.create(collection=collection, url=url, to_delete=True) - # Try the promotion - this should trigger similar conditions to production + # The promotion must apply the deletions without corrupting pattern relations collection.promote_to_curated() - # Print final state for debugging - print("\nFinal state:") + remaining = set(CuratedUrl.objects.filter(collection=collection).values_list("url", flat=True)) + assert remaining == {"https://example.com/docs/guide2", "https://example.com/api/v1/doc2"} for pattern in title_patterns: - print(f"\nPattern '{pattern.match_pattern}':") - for url in pattern.curated_urls.all(): - print(f"- {url.url}") + matched = set(pattern.curated_urls.values_list("url", flat=True)) + assert matched == expected_matches[pattern.match_pattern] & remaining, pattern.match_pattern @pytest.mark.django_db @@ -209,9 +215,14 @@ def test_promotion_with_title_change(): # Now create new DeltaUrl with updated title DeltaUrl.objects.create(collection=collection, url=url, scraped_title="New Title") # Changed title - # This should trigger the same error we're seeing in production + # Regression guard (production bug): updating a CuratedUrl that has an active + # title-pattern relationship must succeed and carry the new title through. collection.promote_to_curated() + curated = CuratedUrl.objects.get(collection=collection, url=url) + assert curated.scraped_title == "New Title" + assert pattern.curated_urls.filter(id=curated.id).exists() + @pytest.mark.django_db def test_promotion_maintains_pattern_relationships_through_updates(collection): diff --git a/sde_collections/tests/test_scrape_dispatch.py b/sde_collections/tests/test_scrape_dispatch.py new file mode 100644 index 00000000..62805711 --- /dev/null +++ b/sde_collections/tests/test_scrape_dispatch.py @@ -0,0 +1,264 @@ +# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_scrape_dispatch.py + +import json +import shlex +from unittest.mock import MagicMock, patch + +import pytest +from django.test import TestCase, override_settings + +from sde_collections.models.collection import Collection +from sde_collections.models.collection_choice_fields import ( + ReindexingStatusChoices, + WorkflowStatusChoices, +) +from sde_collections.models.scraper_config import ScrapeDispatch, ScraperConfigOverride +from sde_collections.scraping.job_builder import build_job_json +from sde_collections.scraping.ssm_dispatch import send_job_to_crawler +from sde_collections.tasks import dispatch_scrape_job +from sde_collections.tests.factories import CollectionFactory + + +@pytest.mark.django_db +class TestBuildJobJson: + def test_no_overrides_emits_exactly_seed_and_collection_id(self): + collection = CollectionFactory() + + job = build_job_json(collection) + + assert job == {"seed": collection.url, "collection_id": collection.config_folder} + + def test_null_override_fields_are_omitted(self): + """crawl4ai's merge_job() skips None — COSMOS must emit only non-null overrides.""" + collection = CollectionFactory() + ScraperConfigOverride.objects.create(collection=collection, max_pages=5000, delay=None) + + job = build_job_json(collection) + + assert job["max_pages"] == 5000 + assert "delay" not in job + assert "depth_limit" not in job + assert "obey_robots" not in job + + def test_false_boolean_override_is_emitted(self): + """False is a real override (e.g. include_subdomains=False), only None is skipped.""" + collection = CollectionFactory() + ScraperConfigOverride.objects.create(collection=collection, include_subdomains=False) + + job = build_job_json(collection) + + assert job["include_subdomains"] is False + + def test_max_pages_above_crawler_cap_raises(self): + collection = CollectionFactory() + ScraperConfigOverride.objects.create(collection=collection, max_pages=200_000) + + with pytest.raises(ValueError, match="100000"): + build_job_json(collection) + + +@pytest.mark.django_db +class TestStatusTriggersDispatch: + # Triggers enqueue on commit; the test transaction never commits, so run the hooks. + @patch("sde_collections.tasks.dispatch_scrape_job.delay") + def test_ready_for_engineering_dispatches_scrape_job(self, mock_delay): + collection = CollectionFactory() + collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING + with TestCase.captureOnCommitCallbacks(execute=True): + collection.save() + + mock_delay.assert_called_once_with(collection.id) + + @patch("sde_collections.tasks.dispatch_scrape_job.delay") + def test_reindexing_needed_dispatches_scrape_job(self, mock_delay): + collection = CollectionFactory() + collection.reindexing_status = ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV + with TestCase.captureOnCommitCallbacks(execute=True): + collection.save() + + mock_delay.assert_called_once_with(collection.id) + + @patch("sde_collections.tasks.dispatch_scrape_job.delay") + def test_ready_for_engineering_no_longer_touches_sinequa_configs(self, mock_delay): + """P6 deleted the Sinequa config methods outright — the trigger can only dispatch.""" + from sde_collections.models.collection import Collection + + collection = CollectionFactory() + collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING + with TestCase.captureOnCommitCallbacks(execute=True): + collection.save() + + mock_delay.assert_called_once_with(collection.id) + assert not hasattr(Collection, "create_scraper_config") + assert not hasattr(Collection, "create_scraper_job") + + +@pytest.mark.django_db +class TestSendJobToCrawler: + """The SSM boundary itself: every caller-side test mocks send_job_to_crawler, so the + script construction (quoting, atomic write, inbox path) must be proven here.""" + + @pytest.fixture(autouse=True) + def crawler_settings(self): + # override_settings can't decorate a plain (non-SimpleTestCase) class + with override_settings(CRAWLER_INSTANCE_ID="i-0test", CRAWLER_INBOX_PATH="/opt/sde-crawler/jobs/incoming"): + yield + + def _dispatch(self, collection): + ssm = MagicMock() + ssm.send_command.return_value = {"Command": {"CommandId": "cmd-xyz"}} + session = MagicMock() + session.client.return_value = ssm + with patch("sde_collections.scraping.ssm_dispatch.get_boto3_session", return_value=session): + command_id = send_job_to_crawler(collection) + return command_id, ssm.send_command.call_args.kwargs + + def test_unconfigured_crawler_refuses_to_dispatch(self): + """A host without the crawler wired must fail loudly on the settings guard rather + than send an SSM command to InstanceIds=[""].""" + collection = CollectionFactory() + with override_settings(CRAWLER_INSTANCE_ID=""): + with pytest.raises(ValueError, match="CRAWLER_INSTANCE_ID"): + send_job_to_crawler(collection) + + def test_command_targets_crawler_instance_and_returns_command_id(self): + collection = CollectionFactory() + + command_id, kwargs = self._dispatch(collection) + + assert command_id == "cmd-xyz" + assert kwargs["InstanceIds"] == ["i-0test"] + assert kwargs["DocumentName"] == "AWS-RunShellScript" + + def test_script_delivers_exact_job_json_via_atomic_rename(self): + """Recover the payload by shell-splitting the script: what the shell would hand to + printf must round-trip back to build_job_json's output, and the write must land on + a .tmp path that is mv'd into the inbox last (the watcher must never see a partial + file).""" + collection = CollectionFactory() + + _, kwargs = self._dispatch(collection) + + lines = kwargs["Parameters"]["commands"][0].splitlines() + dest = f"/opt/sde-crawler/jobs/incoming/{collection.config_folder}.json" + + printf_tokens = shlex.split(lines[0]) + assert printf_tokens[0] == "printf" + assert json.loads(printf_tokens[2]) == build_job_json(collection) + assert printf_tokens[-1] == dest + ".tmp" + + assert shlex.split(lines[1]) == ["chown", "ec2-user:ec2-user", dest + ".tmp"] + assert shlex.split(lines[2]) == ["mv", "-f", dest + ".tmp", dest] + + def test_hostile_seed_url_survives_shell_quoting(self): + """Seed URLs contain quotes/spaces/ampersands that would break an unquoted heredoc.""" + collection = CollectionFactory() + Collection.objects.filter(id=collection.id).update(url="https://example.nasa.gov/search?q='solar wind'&page=1") + collection.refresh_from_db() + + _, kwargs = self._dispatch(collection) + + printf_tokens = shlex.split(kwargs["Parameters"]["commands"][0].splitlines()[0]) + payload = json.loads(printf_tokens[2]) + assert payload["seed"] == "https://example.nasa.gov/search?q='solar wind'&page=1" + + def test_comment_is_truncated_to_ssm_limit(self): + collection = CollectionFactory() + Collection.objects.filter(id=collection.id).update(config_folder="x" * 120) + collection.refresh_from_db() + + _, kwargs = self._dispatch(collection) + + assert len(kwargs["Comment"]) <= 100 + assert kwargs["Comment"].startswith("COSMOS scrape dispatch:") + + +@pytest.mark.django_db +class TestDispatchScrapeJobTask: + @patch("sde_collections.tasks.send_job_to_crawler", return_value="cmd-abc123") + def test_successful_dispatch_records_scrape_dispatch(self, mock_send): + collection = CollectionFactory() + + result = dispatch_scrape_job(collection.id) + + assert result == "cmd-abc123" + dispatch = ScrapeDispatch.objects.get(collection=collection) + assert dispatch.ssm_command_id == "cmd-abc123" + assert dispatch.dispatched_at is not None + + @patch("sde_collections.tasks.notify_status_change") + @patch("sde_collections.tasks.send_job_to_crawler", return_value="cmd-abc123") + def test_successful_dispatch_advances_to_engineering_in_progress(self, mock_send, mock_notify): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + + dispatch_scrape_job(collection.id) + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.ENGINEERING_IN_PROGRESS + history = collection.workflow_history.order_by("-created_at").first() + assert history.workflow_status == WorkflowStatusChoices.ENGINEERING_IN_PROGRESS + assert history.old_status == WorkflowStatusChoices.READY_FOR_ENGINEERING + mock_notify.assert_called_once_with( + collection.name, + collection.id, + WorkflowStatusChoices.READY_FOR_ENGINEERING, + WorkflowStatusChoices.ENGINEERING_IN_PROGRESS, + ) + # Only one dispatch: the status advance must not re-trigger the signal handler. + assert ScrapeDispatch.objects.filter(collection=collection).count() == 1 + + @patch("sde_collections.tasks.notify_status_change") + @patch("sde_collections.tasks.send_job_to_crawler", return_value="cmd-abc123") + def test_rescrape_dispatch_leaves_prod_status_alone(self, mock_send, mock_notify): + """Re-scrape path: the live PROD_* status is not the scrape flow's to rewrite.""" + collection = CollectionFactory( + workflow_status=WorkflowStatusChoices.PROD_PERFECT, + reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV, + ) + + dispatch_scrape_job(collection.id) + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.PROD_PERFECT + mock_notify.assert_not_called() + + @patch("sde_collections.tasks.send_job_to_crawler", side_effect=Exception("SSM unreachable")) + def test_ssm_failure_sets_scraping_failed_and_does_not_raise(self, mock_send): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + + result = dispatch_scrape_job(collection.id) # must not raise + + assert result is None + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.SCRAPING_FAILED + assert not ScrapeDispatch.objects.filter(collection=collection).exists() + + @patch("sde_collections.tasks.send_slack_message") + @patch("sde_collections.tasks.send_job_to_crawler", side_effect=Exception("SSM unreachable")) + def test_rescrape_dispatch_failure_leaves_prod_status_alone(self, mock_send, mock_slack): + """On the re-scrape path the live workflow status (PROD_*) must not be rewritten; + the reindexing request is cleared and an alert is posted instead.""" + collection = CollectionFactory( + workflow_status=WorkflowStatusChoices.PROD_PERFECT, + reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV, + ) + + assert dispatch_scrape_job(collection.id) is None + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.PROD_PERFECT + assert collection.reindexing_status == ReindexingStatusChoices.REINDEXING_NOT_NEEDED + mock_slack.assert_called_once() + + @patch("sde_collections.tasks.send_job_to_crawler", return_value="cmd-2") + def test_redispatch_keeps_prior_dispatch_rows(self, mock_send): + """Rows are never deleted — the poller takes the latest per collection.""" + collection = CollectionFactory() + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + + dispatch_scrape_job(collection.id) + + ids = list(ScrapeDispatch.objects.filter(collection=collection).values_list("ssm_command_id", flat=True)) + assert set(ids) == {"cmd-1", "cmd-2"} + # Meta.ordering is -dispatched_at: first row is the latest dispatch + assert ScrapeDispatch.objects.filter(collection=collection).first().ssm_command_id == "cmd-2" diff --git a/sde_collections/tests/test_scrape_ingest.py b/sde_collections/tests/test_scrape_ingest.py new file mode 100644 index 00000000..74b107e4 --- /dev/null +++ b/sde_collections/tests/test_scrape_ingest.py @@ -0,0 +1,417 @@ +# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_scrape_ingest.py + +import io +import json +from datetime import datetime, timedelta +from datetime import timezone as dt_timezone +from unittest.mock import MagicMock, patch + +import pytest +from botocore.exceptions import ClientError +from django.test import override_settings +from django.utils import timezone + +from sde_collections.models.collection_choice_fields import ( + ReindexingStatusChoices, + WorkflowStatusChoices, +) +from sde_collections.models.delta_url import DumpUrl +from sde_collections.models.scraper_config import ScrapeDispatch +from sde_collections.scraping.s3_results import ( + fetch_documents, + fetch_summary, + results_ready, +) +from sde_collections.tasks import ( + ingest_scraped_collection, + migrate_dump_to_delta_and_handle_status_transistions, + poll_scrape_jobs, +) +from sde_collections.tests.factories import CollectionFactory + + +def make_documents(count, seed="https://example.nasa.gov/"): + """Fixture matching the crawler's exact 7-field document shape.""" + return [ + { + "url": f"{seed}page-{i}", + "title": f"Page {i}", + "full_text": f"Full text of page {i}", + "content_type": "text/html", + "seed": seed, + "host": "example.nasa.gov", + "depth": 1, + } + for i in range(count) + ] + + +def make_summary(documents_scraped): + return {"documents_scraped": documents_scraped, "failures_logged": 0, "seed": "https://example.nasa.gov/"} + + +def backdate_dispatch(dispatch, hours): + ScrapeDispatch.objects.filter(id=dispatch.id).update(dispatched_at=timezone.now() - timedelta(hours=hours)) + + +@pytest.mark.django_db +class TestIngestScrapedCollection: + @patch("sde_collections.tasks.migrate_dump_to_delta_and_handle_status_transistions.delay") + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.results_ready") + def test_happy_path_creates_dump_urls_and_claims_status(self, mock_ready, mock_docs, mock_migrate): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(3) + mock_docs.return_value = make_documents(3) + + ingest_scraped_collection(collection.id) + + dumps = DumpUrl.objects.filter(collection=collection).order_by("url") + assert dumps.count() == 3 + assert dumps[0].scraped_title == "Page 0" + assert dumps[0].scraped_text == "Full text of page 0" + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.SCRAPING_SUCCESSFUL + mock_migrate.assert_called_once_with(collection.id) + + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.results_ready") + def test_zero_documents_marks_scraping_failed(self, mock_ready, mock_docs): + """A zero-page crawl otherwise 'succeeds' and silently publishes an empty collection.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(0) + + ingest_scraped_collection(collection.id) + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.SCRAPING_FAILED + assert not DumpUrl.objects.filter(collection=collection).exists() + mock_docs.assert_not_called() + + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.results_ready") + def test_concurrent_claim_exits_without_touching_dump_urls(self, mock_ready, mock_docs): + """CAS returns 0 for an already-claimed collection — second ingest must exit.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.SCRAPING_SUCCESSFUL) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + DumpUrl.objects.create(collection=collection, url="https://example.nasa.gov/existing") + mock_ready.return_value = make_summary(3) + + result = ingest_scraped_collection(collection.id) + + assert "already claimed" in result + assert DumpUrl.objects.filter(collection=collection).count() == 1 + mock_docs.assert_not_called() + + @patch("sde_collections.tasks.migrate_dump_to_delta_and_handle_status_transistions.delay") + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.fetch_summary") + def test_manual_ingest_is_idempotent(self, mock_summary, mock_docs, mock_migrate): + """Re-running manual ingest yields N (not 2N) DumpUrls — delete-then-write.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.SCRAPING_SUCCESSFUL) + mock_summary.return_value = (make_summary(4), datetime.now(dt_timezone.utc)) + mock_docs.return_value = make_documents(4) + + ingest_scraped_collection(collection.id, claim=False) + ingest_scraped_collection(collection.id, claim=False) + + assert DumpUrl.objects.filter(collection=collection).count() == 4 + + @patch("sde_collections.tasks.fetch_documents", side_effect=Exception("S3 read died")) + @patch("sde_collections.tasks.results_ready") + def test_ingest_failure_after_claim_sets_scraping_failed(self, mock_ready, mock_docs): + """Never leave a claimed collection stuck in SCRAPING_SUCCESSFUL with no DumpUrls.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(3) + + result = ingest_scraped_collection(collection.id) # must not raise + + assert result is None + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.SCRAPING_FAILED + + @patch("sde_collections.tasks.notify_status_change") + @patch("sde_collections.tasks.migrate_dump_to_delta_and_handle_status_transistions.delay") + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.results_ready") + def test_claim_posts_the_scraping_successful_notification(self, mock_ready, mock_docs, mock_migrate, mock_notify): + """The claim is a queryset .update() (no post_save), so the task must post the + READY_FOR_ENGINEERING -> SCRAPING_SUCCESSFUL message itself.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(1) + mock_docs.return_value = make_documents(1) + + ingest_scraped_collection(collection.id) + + mock_notify.assert_called_once_with( + collection.name, + collection.id, + WorkflowStatusChoices.READY_FOR_ENGINEERING, + WorkflowStatusChoices.SCRAPING_SUCCESSFUL, + ) + # SCRAPING_SUCCESSFUL is short-lived (migration follows within seconds); the + # timeline row is the only durable evidence it happened. + history = collection.workflow_history.order_by("-created_at").first() + assert history.workflow_status == WorkflowStatusChoices.SCRAPING_SUCCESSFUL + assert history.old_status == WorkflowStatusChoices.READY_FOR_ENGINEERING + + @patch("sde_collections.tasks.send_slack_message") + @patch("sde_collections.tasks.fetch_documents", side_effect=Exception("S3 read died")) + @patch("sde_collections.tasks.results_ready") + def test_rescrape_ingest_failure_leaves_workflow_status_alone(self, mock_ready, mock_docs, mock_slack): + """A re-scrape hiccup must never rewrite a production collection's workflow status.""" + collection = CollectionFactory( + workflow_status=WorkflowStatusChoices.PROD_PERFECT, + reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV, + ) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(2) + + assert ingest_scraped_collection(collection.id) is None + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.PROD_PERFECT + # Cleared so the poller does not re-enqueue the same failed run every 5 minutes. + assert collection.reindexing_status == ReindexingStatusChoices.REINDEXING_NOT_NEEDED + mock_slack.assert_called_once() + + @patch("sde_collections.tasks.migrate_dump_to_delta_and_handle_status_transistions.delay") + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.results_ready") + def test_duplicate_urls_in_crawl_output_are_dropped(self, mock_ready, mock_docs, mock_migrate): + """BaseUrl.url is unique: one repeated URL must not abort the whole bulk_create.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + documents = make_documents(3) + documents.append(dict(documents[0], title="Duplicate of page 0")) + documents.append({"url": "", "title": "blank url"}) + mock_ready.return_value = make_summary(len(documents)) + mock_docs.return_value = documents + + ingest_scraped_collection(collection.id) + + dumps = DumpUrl.objects.filter(collection=collection) + assert dumps.count() == 3 + assert dumps.get(url=documents[0]["url"]).scraped_title == "Page 0" # first occurrence wins + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.SCRAPING_SUCCESSFUL + + @patch("sde_collections.tasks.send_detailed_import_notification") + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.results_ready") + def test_rescrape_path_ends_ready_for_recuration(self, mock_ready, mock_docs, mock_slack): + """reindexing NEEDED -> ingest claims to FINISHED -> migrate task promotes to + REINDEXING_READY_FOR_CURATION (the migrate task's existing transition).""" + collection = CollectionFactory( + workflow_status=WorkflowStatusChoices.PROD_PERFECT, + reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV, + ) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(2) + mock_docs.return_value = make_documents(2) + + with patch( + "sde_collections.models.collection.migrate_dump_to_delta_and_handle_status_transistions.delay" + ) as mock_delay: + ingest_scraped_collection(collection.id) + + collection.refresh_from_db() + assert collection.reindexing_status == ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV + assert collection.workflow_status == WorkflowStatusChoices.PROD_PERFECT # untouched + mock_delay.assert_called_once_with(collection.id) + + # Now run the migration task for real (the .delay was mocked above). + migrate_dump_to_delta_and_handle_status_transistions(collection.id) + + collection.refresh_from_db() + assert collection.reindexing_status == ReindexingStatusChoices.REINDEXING_READY_FOR_CURATION + mock_slack.assert_called_once() + + @patch("sde_collections.tasks.send_detailed_import_notification") + @patch("sde_collections.tasks.fetch_documents") + @patch("sde_collections.tasks.results_ready") + def test_end_to_end_reaches_ready_for_curation_with_inference_off(self, mock_ready, mock_docs, mock_slack): + """P2+P4 integration: ingest -> SCRAPING_SUCCESSFUL -> migrate -> READY_FOR_CURATION, + with the ingest summary posted to Slack.""" + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(3) + mock_docs.return_value = make_documents(3) + + with patch( + "sde_collections.models.collection.migrate_dump_to_delta_and_handle_status_transistions.delay" + ) as mock_delay: + ingest_scraped_collection(collection.id) + mock_delay.assert_called_once_with(collection.id) + + migrate_dump_to_delta_and_handle_status_transistions(collection.id) + + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.READY_FOR_CURATION + assert collection.delta_urls.count() == 3 + mock_slack.assert_called_once() + assert mock_slack.call_args.kwargs["dump_count"] == 3 + assert mock_slack.call_args.kwargs["delta_count"] == 3 + + +@pytest.mark.django_db +class TestPollScrapeJobs: + @patch("sde_collections.tasks.ingest_scraped_collection.delay") + @patch("sde_collections.tasks.results_ready") + def test_fresh_results_enqueue_ingest(self, mock_ready, mock_ingest): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(3) + + poll_scrape_jobs() + + mock_ingest.assert_called_once_with(collection.id) + + @patch("sde_collections.tasks.ingest_scraped_collection.delay") + @patch("sde_collections.tasks.results_ready", return_value=None) + def test_missing_summary_leaves_collection_waiting(self, mock_ready, mock_ingest): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + + poll_scrape_jobs() + + mock_ingest.assert_not_called() + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING + + @patch("sde_collections.tasks.ingest_scraped_collection.delay") + @patch("sde_collections.tasks.results_ready", return_value=None) + def test_stall_timeout_marks_scraping_failed(self, mock_ready, mock_ingest): + collection = CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + dispatch = ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + backdate_dispatch(dispatch, hours=25) # default timeout is 24h + + poll_scrape_jobs() + + mock_ingest.assert_not_called() + collection.refresh_from_db() + assert collection.workflow_status == WorkflowStatusChoices.SCRAPING_FAILED + + @patch("sde_collections.tasks.ingest_scraped_collection.delay") + @patch("sde_collections.tasks.results_ready") + def test_undispatched_collection_is_skipped(self, mock_ready, mock_ingest): + CollectionFactory(workflow_status=WorkflowStatusChoices.READY_FOR_ENGINEERING) + + poll_scrape_jobs() + + mock_ready.assert_not_called() + mock_ingest.assert_not_called() + + @patch("sde_collections.tasks.ingest_scraped_collection.delay") + @patch("sde_collections.tasks.results_ready") + def test_rescrape_collections_are_polled(self, mock_ready, mock_ingest): + collection = CollectionFactory( + workflow_status=WorkflowStatusChoices.PROD_PERFECT, + reindexing_status=ReindexingStatusChoices.REINDEXING_NEEDED_ON_DEV, + ) + ScrapeDispatch.objects.create(collection=collection, ssm_command_id="cmd-1") + mock_ready.return_value = make_summary(2) + + poll_scrape_jobs() + + mock_ingest.assert_called_once_with(collection.id) + + +def _s3_stub(get_object=None): + """Patchable session whose s3 client's get_object returns (or raises) the given value.""" + s3 = MagicMock() + if isinstance(get_object, BaseException): + s3.get_object.side_effect = get_object + else: + s3.get_object.return_value = get_object + session = MagicMock() + session.client.return_value = s3 + return session, s3 + + +def _client_error(code): + return ClientError({"Error": {"Code": code}}, "GetObject") + + +def _s3_object(payload, last_modified=None): + return { + "Body": io.BytesIO(json.dumps(payload).encode("utf-8")), + "LastModified": last_modified or datetime(2026, 8, 13, 12, 0, 0, tzinfo=dt_timezone.utc), + } + + +class TestS3ResultFetchers: + """The S3 boundary itself: the ingest/poll tests all mock fetch_summary/fetch_documents, + so the real key layout, bucket wiring and missing-key handling must be proven here. + The keys are the crawler's contract (sde_crawler/job.py::s3_keys_for_collection).""" + + @pytest.fixture(autouse=True) + def crawler_bucket(self): + # override_settings can't decorate a plain (non-SimpleTestCase) class + with override_settings(SDE_S3_BUCKET="crawler-bucket-test"): + yield + + def test_fetch_summary_reads_crawler_summary_key_and_returns_last_modified(self): + summary = make_summary(7) + modified = datetime(2026, 8, 14, 9, 30, 0, tzinfo=dt_timezone.utc) + session, s3 = _s3_stub(_s3_object(summary, modified)) + + with patch("sde_collections.scraping.s3_results.get_boto3_session", return_value=session): + result = fetch_summary("astro_data") + + assert result == (summary, modified) + s3.get_object.assert_called_once_with( + Bucket="crawler-bucket-test", Key="failure_logs/astro_data_failures_summary.json" + ) + + @pytest.mark.parametrize("code", ["NoSuchKey", "404"]) + def test_fetch_summary_missing_key_means_run_not_finished(self, code): + session, _ = _s3_stub(_client_error(code)) + + with patch("sde_collections.scraping.s3_results.get_boto3_session", return_value=session): + assert fetch_summary("astro_data") is None + + def test_fetch_summary_propagates_real_s3_errors(self): + """AccessDenied etc. must surface, not read as 'still crawling' forever.""" + session, _ = _s3_stub(_client_error("AccessDenied")) + + with patch("sde_collections.scraping.s3_results.get_boto3_session", return_value=session): + with pytest.raises(ClientError): + fetch_summary("astro_data") + + def test_fetch_documents_reads_scraped_collections_key(self): + documents = make_documents(2) + session, s3 = _s3_stub(_s3_object(documents)) + + with patch("sde_collections.scraping.s3_results.get_boto3_session", return_value=session): + result = fetch_documents("astro_data") + + assert result == documents + s3.get_object.assert_called_once_with(Bucket="crawler-bucket-test", Key="scraped_collections/astro_data.json") + + +class TestResultsFreshness: + """Stale-results regression guard: a summary older than the dispatch is treated as absent.""" + + @patch("sde_collections.scraping.s3_results.fetch_summary") + def test_stale_summary_is_treated_as_absent(self, mock_fetch): + dispatched_at = datetime(2026, 8, 13, 12, 0, 0, tzinfo=dt_timezone.utc) + mock_fetch.return_value = (make_summary(5), dispatched_at - timedelta(hours=2)) + + assert results_ready("some_collection", dispatched_at) is None + + @patch("sde_collections.scraping.s3_results.fetch_summary") + def test_fresh_summary_is_returned(self, mock_fetch): + dispatched_at = datetime(2026, 8, 13, 12, 0, 0, tzinfo=dt_timezone.utc) + summary = make_summary(5) + mock_fetch.return_value = (summary, dispatched_at + timedelta(minutes=10)) + + assert results_ready("some_collection", dispatched_at) == summary + + @patch("sde_collections.scraping.s3_results.fetch_summary", return_value=None) + def test_missing_summary_returns_none(self, mock_fetch): + assert results_ready("some_collection", timezone.now()) is None diff --git a/sde_collections/tests/test_signals.py b/sde_collections/tests/test_signals.py new file mode 100644 index 00000000..e9546dac --- /dev/null +++ b/sde_collections/tests/test_signals.py @@ -0,0 +1,66 @@ +# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_signals.py + +from types import SimpleNamespace + +import pytest +from django.test import override_settings + +from sde_collections.signals import ( + POLL_INDEX_TASK, + POLL_INDEX_TASK_NAME, + POLL_SCRAPE_TASK, + POLL_SCRAPE_TASK_NAME, + create_periodic_tasks, +) + + +def _run_post_migrate_handler(app_label="sde_collections"): + create_periodic_tasks(sender=SimpleNamespace(name=app_label)) + + +def _rows(): + from django_celery_beat.models import PeriodicTask + + return {row.name: row for row in PeriodicTask.objects.filter(task__in=[POLL_SCRAPE_TASK, POLL_INDEX_TASK])} + + +@pytest.mark.django_db +@pytest.mark.parametrize("scrape_flag,index_flag", [(False, False), (True, False), (False, True), (True, True)]) +def test_poller_rows_follow_their_flags(scrape_flag, index_flag): + with override_settings(SCRAPE_POLL_ENABLED=scrape_flag, INDEX_POLL_ENABLED=index_flag): + _run_post_migrate_handler() + + rows = _rows() + assert rows[POLL_SCRAPE_TASK_NAME].enabled is scrape_flag + assert rows[POLL_SCRAPE_TASK_NAME].crontab.minute == "*/5" + assert rows[POLL_INDEX_TASK_NAME].enabled is index_flag + assert rows[POLL_INDEX_TASK_NAME].crontab.minute == "*/2" + + +@pytest.mark.django_db +@override_settings(SCRAPE_POLL_ENABLED=False, INDEX_POLL_ENABLED=False) +def test_flag_is_reasserted_on_every_migrate(): + """The flag, not an admin hand-edit, is the source of truth: a hand-enable is + reverted by the next deploy's migrate step (and vice versa).""" + from django_celery_beat.models import PeriodicTask + + _run_post_migrate_handler() + PeriodicTask.objects.filter(task__in=[POLL_SCRAPE_TASK, POLL_INDEX_TASK]).update(enabled=True) + + _run_post_migrate_handler() + + assert all(row.enabled is False for row in _rows().values()) + assert len(_rows()) == 2 # updated in place, never duplicated + + +@pytest.mark.django_db +@override_settings(SCRAPE_POLL_ENABLED=True, INDEX_POLL_ENABLED=True) +def test_handler_ignores_other_apps(): + from django_celery_beat.models import PeriodicTask + + # Test-DB setup already ran the real post_migrate; start from a clean slate. + PeriodicTask.objects.filter(task__in=[POLL_SCRAPE_TASK, POLL_INDEX_TASK]).delete() + + _run_post_migrate_handler(app_label="inference") + + assert _rows() == {} diff --git a/sde_collections/tests/test_sinequa_api.py b/sde_collections/tests/test_sinequa_api.py deleted file mode 100644 index 51c75f36..00000000 --- a/sde_collections/tests/test_sinequa_api.py +++ /dev/null @@ -1,364 +0,0 @@ -# docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_sinequa_api.py -import json -from unittest.mock import MagicMock, patch - -import pytest -import requests -from django.utils import timezone - -from sde_collections.models.collection import WorkflowStatusChoices -from sde_collections.sinequa_api import Api -from sde_collections.tests.factories import CollectionFactory, UserFactory - - -@pytest.mark.django_db -class TestApiClass: - """ - Test suite for the Sinequa API integration. - Tests cover authentication, query construction, response processing, - and error handling across different server configurations. - """ - - @pytest.fixture - def collection(self): - """Fixture to create a collection object for testing.""" - user = UserFactory() - return CollectionFactory( - curated_by=user, - curation_started=timezone.now(), - config_folder="example_config", - workflow_status=WorkflowStatusChoices.RESEARCH_IN_PROGRESS, - ) - - @pytest.fixture - def api_instance(self): - """ - Fixture to create an Api instance with mocked server configs. - Provides a consistent test environment with predefined credentials. - """ - with patch( - "sde_collections.sinequa_api.server_configs", - { - "test_server": { - "app_name": "test_app", - "query_name": "test_query", - "base_url": "http://testserver.com/api", - "index": "test_index", - } - }, - ): - return Api(server_name="test_server", user="test_user", password="test_pass", token="test_token") - - @patch("requests.post") - def test_process_response_success(self, mock_post, api_instance): - """ - Test that process_response successfully handles and parses API responses. - Verifies: - 1. Correct HTTP request processing - 2. JSON response parsing - 3. Return value structure - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"key": "value"} - mock_post.return_value = mock_response - - response = api_instance.process_response("http://example.com", payload={"test": "data"}) - assert response == {"key": "value"} - mock_post.assert_called_once() - - @patch("requests.post") - def test_process_response_failure(self, mock_post, api_instance): - """ - Test that process_response properly handles failed API requests. - Verifies appropriate exception raising and error messaging. - """ - mock_response = MagicMock() - mock_response.status_code = 500 - mock_post.return_value = mock_response - mock_response.raise_for_status.side_effect = requests.RequestException("Internal Server Error") - - with pytest.raises(requests.RequestException, match="Internal Server Error"): - api_instance.process_response("http://example.com", payload={"test": "data"}) - - def test_missing_token_for_sql_query(self, api_instance): - """ - Test that attempting SQL queries without a token raises an appropriate error. - Verifies token validation before query execution. - """ - api_instance._provided_token = None - with pytest.raises(ValueError, match="Token is required"): - api_instance._execute_sql_query("SELECT * FROM test") - - @patch("sde_collections.sinequa_api.Api.process_response") - def test_query(self, mock_process_response, api_instance): - """ - Test that query method: - 1. Constructs the correct URL and payload based on input parameters - 2. Processes API response correctly - 3. Returns expected data structure - """ - mock_process_response.return_value = {"result": "success"} - response = api_instance.query(page=1, collection_config_folder="folder") - assert response == {"result": "success"} - - # Verify payload construction - mock_process_response.assert_called_once() - call_args = mock_process_response.call_args - assert "folder" in str(call_args) # Verify collection folder is included - assert "page" in str(call_args) # Verify pagination parameters - - def test_process_rows_to_records(self, api_instance): - """ - Test processing of raw SQL row data into structured record dictionaries. - Verifies: - 1. Correct parsing of valid input data - 2. Error handling for malformed rows - 3. Output format consistency - """ - # Test valid input - valid_rows = [["http://example.com/1", "Text 1", "Title 1"], ["http://example.com/2", "Text 2", "Title 2"]] - expected_output = [ - {"url": "http://example.com/1", "full_text": "Text 1", "title": "Title 1"}, - {"url": "http://example.com/2", "full_text": "Text 2", "title": "Title 2"}, - ] - assert api_instance._process_rows_to_records(valid_rows) == expected_output - - # Test invalid row length - invalid_rows = [["http://example.com", "Text"]] # Missing title - with pytest.raises(ValueError, match="Invalid row format at index 0"): - api_instance._process_rows_to_records(invalid_rows) - - @patch("sde_collections.sinequa_api.Api.process_response") - def test_execute_sql_query(self, mock_process_response, api_instance): - """ - Test SQL query execution with token-based authentication. - Verifies: - 1. Query construction - 2. Token validation - 3. Response processing - """ - mock_process_response.return_value = {"Rows": [], "TotalRowCount": 0} - - # Test successful query - result = api_instance._execute_sql_query("SELECT * FROM test") - assert result == {"Rows": [], "TotalRowCount": 0} - - # Test query with missing token - api_instance._provided_token = None - with pytest.raises(ValueError, match="Token is required"): - api_instance._execute_sql_query("SELECT * FROM test") - - @patch("sde_collections.sinequa_api.Api._execute_sql_query") - def test_get_full_texts_pagination(self, mock_execute_sql, api_instance): - """ - Test pagination handling in get_full_texts method. - Verifies: - 1. Correct batch processing - 2. Accurate record counting - 3. Proper iteration termination - """ - # Mock responses for two pages of results - mock_execute_sql.side_effect = [ - { - "Rows": [["http://example.com/1", "Text 1", "Title 1"], ["http://example.com/2", "Text 2", "Title 2"]], - "TotalRowCount": 3, - }, - {"Rows": [["http://example.com/3", "Text 3", "Title 3"]], "TotalRowCount": 3}, - {"Rows": [], "TotalRowCount": 3}, - ] - - # Collect all batches from the iterator - batches = list(api_instance.get_full_texts("test_folder")) - assert len(batches) == 2 # Should have two batches - assert len(batches[0]) == 2 # First batch has 2 records - assert len(batches[1]) == 1 # Second batch has 1 record - - # Verify content of batches - assert batches[0] == [ - {"url": "http://example.com/1", "full_text": "Text 1", "title": "Title 1"}, - {"url": "http://example.com/2", "full_text": "Text 2", "title": "Title 2"}, - ] - assert batches[1] == [{"url": "http://example.com/3", "full_text": "Text 3", "title": "Title 3"}] - - def test_get_full_texts_missing_index(self, api_instance): - """ - Test error handling when index configuration is missing. - Verifies appropriate error message and exception type. - """ - api_instance.config.pop("index", None) - with pytest.raises(ValueError, match="Index not defined for server"): - next(api_instance.get_full_texts("test_folder")) - - @pytest.mark.parametrize( - "server_name,expect_auth", - [ - ("xli", True), # dev server should have auth - ("production", False), # prod server should not have auth - ], - ) - @patch("requests.post") - def test_query_authentication(self, mock_post, server_name, expect_auth, api_instance): - """ - Test authentication handling for different server types. - Verifies: - 1. Dev servers require authentication - 2. Production servers skip authentication - 3. Correct credential handling - """ - api_instance.server_name = server_name - mock_post.return_value = MagicMock(status_code=200, json=lambda: {"result": "success"}) - - response = api_instance.query(page=1, collection_config_folder="folder") - assert response == {"result": "success"} - - called_url = mock_post.call_args[0][0] - auth_present = "?Password=test_pass&User=test_user" in called_url - assert auth_present == expect_auth - - @patch("requests.post") - def test_query_dev_server_missing_credentials(self, mock_post, api_instance): - """ - Test error handling for dev servers with missing credentials. - Verifies appropriate error messages and authentication requirements. - """ - api_instance.server_name = "xli" - api_instance._provided_user = None - api_instance._provided_password = None - - with pytest.raises(ValueError, match="Authentication error: Missing credentials for dev server"): - api_instance.query(page=1) - - @patch("sde_collections.sinequa_api.Api._execute_sql_query") - def test_get_full_texts_batch_size_reduction(self, mock_execute_sql, api_instance): - """ - Test batch size reduction logic when queries fail. - Verifies: - 1. Progressive batch size reduction - 2. Retry mechanism - 3. Successful recovery - """ - # Mock first query to fail, then succeed with smaller batch - mock_execute_sql.side_effect = [ - requests.RequestException("Query too large"), # First attempt fails - { - "Rows": [["http://example.com/1", "Text 1", "Title 1"]], - "TotalRowCount": 1, - }, # Succeeds with smaller batch - ] - - batches = list(api_instance.get_full_texts("test_folder", batch_size=100, min_batch_size=1)) - - # Verify the batches were processed correctly after size reduction - assert len(batches) == 1 - assert len(batches[0]) == 1 - assert batches[0][0]["url"] == "http://example.com/1" - - # Verify batch size reduction logic - assert mock_execute_sql.call_count == 2 - first_call = mock_execute_sql.call_args_list[0][0][0] - second_call = mock_execute_sql.call_args_list[1][0][0] - assert "COUNT 100" in first_call - assert "COUNT 50" in second_call # Should be halved from 100 - - @patch("sde_collections.sinequa_api.Api._execute_sql_query") - def test_get_full_texts_minimum_batch_size(self, mock_execute_sql, api_instance): - """ - Test behavior when reaching minimum batch size. - Verifies error handling at minimum batch size threshold. - """ - mock_execute_sql.side_effect = requests.RequestException("Query failed") - - # Start with batch_size=4, min_batch_size=1 - with pytest.raises(ValueError, match="Failed to process batch even at minimum size 1"): - list(api_instance.get_full_texts("test_folder", batch_size=4, min_batch_size=1)) - - # Verify retry attempts - assert mock_execute_sql.call_count == 3 - calls = mock_execute_sql.call_args_list - assert "COUNT 4" in calls[0][0][0] # First try with 4 - assert "COUNT 2" in calls[1][0][0] # Second try with 2 - assert "COUNT 1" in calls[2][0][0] # Final try with 1 - - @patch("requests.post") - def test_sql_query_construction(self, mock_post, api_instance): - """ - Test direct SQL query execution with specific URL and payload validation. - Verifies: - 1. Correct URL construction - 2. Proper payload formatting - 3. Token-based authentication - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"Rows": [["http://example.com", "sample text", "sample title"]]} - mock_post.return_value = mock_response - - sql = "SELECT url1, text, title FROM test_index WHERE collection = '/SDE/sample_folder/'" - api_instance._execute_sql_query(sql) - - # Verify URL and payload construction - mock_post.assert_called_once() - call_args = mock_post.call_args - - # Get the actual payload from the call arguments - _, kwargs = call_args - payload = json.loads(kwargs.get("data", "{}")) - - # Verify each component separately - assert "engine.sql" in call_args[0][0] # Verify endpoint - assert kwargs["headers"]["Authorization"] == "Bearer test_token" # Verify token usage - assert payload["sql"] == sql # Verify SQL query inclusion - - def test_process_full_text_response(self, api_instance): - """ - Test static method for processing full text response data. - Verifies: - 1. Correct parsing of raw response data - 2. Proper dictionary structure creation - 3. Error handling for invalid response format - """ - # Test valid response processing - raw_response = { - "Rows": [ - ["http://example.com/article1", "Full text 1", "Title 1"], - ["http://example.com/article2", "Full text 2", "Title 2"], - ] - } - expected = [ - {"url": "http://example.com/article1", "full_text": "Full text 1", "title": "Title 1"}, - {"url": "http://example.com/article2", "full_text": "Full text 2", "title": "Title 2"}, - ] - processed = Api._process_full_text_response(raw_response) - assert processed == expected - - # Test invalid response format - with pytest.raises(ValueError, match="Invalid response format"): - Api._process_full_text_response({"wrong_key": []}) - - @patch("sde_collections.sinequa_api.Api._execute_sql_query") - def test_get_full_texts_batch_size_progression(self, mock_execute_sql, api_instance): - """ - Test multiple batch size reductions followed by successful query. - Verifies: - 1. Progressive batch size reduction steps - 2. Recovery after multiple failures - 3. Final successful query execution - """ - mock_execute_sql.side_effect = [ - requests.RequestException("First failure"), - requests.RequestException("Second failure"), - {"Rows": [["http://example.com/1", "Text 1", "Title 1"]], "TotalRowCount": 1}, - ] - - # Start with batch_size=100, should reduce to 25 before succeeding - batches = list(api_instance.get_full_texts("test_folder", batch_size=100, min_batch_size=1)) - - assert len(batches) == 1 # Should get one successful batch - assert mock_execute_sql.call_count == 3 - - calls = mock_execute_sql.call_args_list - # Verify the progression of batch sizes - assert "COUNT 100" in calls[0][0][0] # First attempt - assert "COUNT 50" in calls[1][0][0] # After first failure - assert "COUNT 25" in calls[2][0][0] # After second failure diff --git a/sde_collections/tests/test_url_apis.py b/sde_collections/tests/test_url_apis.py index 8ce4d93a..2a216a1d 100644 --- a/sde_collections/tests/test_url_apis.py +++ b/sde_collections/tests/test_url_apis.py @@ -221,22 +221,25 @@ def test_candidate_url_api_serializer_fields(self, client): assert set(data.keys()) == expected_fields def test_candidate_url_api_alias(self, client): - """Should verify candidate-urls-api endpoint aliases to candidate-urls-api""" - candidate_url = CuratedUrlFactory(collection=self.collection, generated_title="Test Generated Title") + """The legacy candidate-urls-api route must serve the exact same payload as + curated-urls-api (both are wired to CuratedURLAPIView).""" + CuratedUrlFactory(collection=self.collection, generated_title="Test Generated Title") candidate_url = reverse( "sde_collections:candidate-url-api", kwargs={"config_folder": self.collection.config_folder} ) - candidate_url = reverse( - "sde_collections:candidate-url-api", kwargs={"config_folder": self.collection.config_folder} + curated_url = reverse( + "sde_collections:curated-url-api", kwargs={"config_folder": self.collection.config_folder} ) + assert candidate_url != curated_url # distinct routes, same view candidate_response = client.get(candidate_url) - candidate_response = client.get(candidate_url) + curated_response = client.get(curated_url) assert candidate_response.status_code == status.HTTP_200_OK - assert candidate_response.status_code == status.HTTP_200_OK - assert candidate_response.json()["results"] == candidate_response.json()["results"] + assert curated_response.status_code == status.HTTP_200_OK + assert len(candidate_response.json()["results"]) == 1 + assert candidate_response.json()["results"] == curated_response.json()["results"] def test_multiple_collections(self, client): """Should only return URLs from the specified collection""" diff --git a/sde_collections/tests/test_workflow_status_triggers.py b/sde_collections/tests/test_workflow_status_triggers.py index b66ff0f6..46b44d20 100644 --- a/sde_collections/tests/test_workflow_status_triggers.py +++ b/sde_collections/tests/test_workflow_status_triggers.py @@ -1,483 +1,269 @@ # docker-compose -f local.yml run --rm django pytest sde_collections/tests/test_workflow_status_triggers.py -from unittest.mock import Mock, patch +from unittest.mock import patch import pytest -from django.db import transaction -from django.test import TestCase, TransactionTestCase +from django.test import TestCase +from sde_collections.models.collection import Collection, WorkflowHistory from sde_collections.models.collection_choice_fields import ( ReindexingStatusChoices, WorkflowStatusChoices, ) -from sde_collections.models.delta_url import DumpUrl -from sde_collections.tasks import ( - fetch_full_text, - migrate_dump_to_delta_and_handle_status_transistions, -) -from sde_collections.tests.factories import CollectionFactory, DumpUrlFactory +from sde_collections.tests.factories import CollectionFactory class TestWorkflowStatusTransitions(TestCase): + """P5 trigger table: the dispatcher (handle_workflow_status_change) drives only the + crawl4ai/indexing pipeline — every Sinequa branch is gone.""" + def setUp(self): self.collection = CollectionFactory() - @patch("sde_collections.models.collection.GitHubHandler") - @patch("sde_collections.models.collection.Collection.create_scraper_config") - def test_ready_for_engineering_triggers_config_creation(self, mock_scraper, mock_github_handler): - """When status changes to READY_FOR_ENGINEERING, it should create configs""" + @patch("sde_collections.tasks.dispatch_scrape_job.delay") + def test_ready_for_engineering_triggers_scrape_dispatch(self, mock_dispatch): + """When status changes to READY_FOR_ENGINEERING, it should dispatch a scrape job (P3: + replaces the Sinequa create_scraper_config/create_scraper_job trigger).""" self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING - self.collection.save() + with self.captureOnCommitCallbacks(execute=True): + self.collection.save() - mock_scraper.assert_called_once_with(overwrite=False) + mock_dispatch.assert_called_once_with(self.collection.id) - @patch("sde_collections.tasks.fetch_full_text.delay") - def test_indexing_finished_triggers_full_text_fetch(self, mock_fetch): - """When status changes to INDEXING_FINISHED_ON_DEV, it should trigger full text fetch""" - self.collection.workflow_status = WorkflowStatusChoices.INDEXING_FINISHED_ON_DEV - self.collection.save() - - mock_fetch.assert_called_once_with(self.collection.id, "lrm_dev") - - @patch("sde_collections.models.collection.Collection.create_indexer_config") - @patch("sde_collections.models.collection.GitHubHandler") - def test_ready_for_curation_triggers_indexer_config(self, mock_github_handler, mock_indexer): - """When status changes to READY_FOR_CURATION, it should create indexer config""" - self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION - self.collection.save() - - mock_indexer.assert_called_once_with(overwrite=True) + @patch("sde_collections.tasks.dispatch_scrape_job.delay") + def test_task_is_enqueued_only_after_commit(self, mock_dispatch): + """Tasks are enqueued via transaction.on_commit so a worker can never read the + collection (or promote_to_curated's rows) before the triggering save commits.""" + self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING + with self.captureOnCommitCallbacks() as callbacks: + self.collection.save() + mock_dispatch.assert_not_called() + + assert len(callbacks) == 1 + callbacks[0]() + mock_dispatch.assert_called_once_with(self.collection.id) + + def _save_and_assert_no_triggers(self, **status_updates): + """Set the given status field(s), save, and assert none of the surviving + pipeline triggers fire. (The Sinequa methods can't be patched — they no + longer exist; test_no_sinequa_methods_remain proves that directly.)""" + with ( + patch("sde_collections.tasks.dispatch_scrape_job.delay") as mock_dispatch, + patch("sde_collections.tasks.index_collection_to_test.delay") as mock_test, + patch("sde_collections.tasks.index_collection_to_prod.delay") as mock_prod, + patch("sde_collections.models.collection.Collection.promote_to_curated") as mock_promote, + ): + for field, value in status_updates.items(): + setattr(self.collection, field, value) + with self.captureOnCommitCallbacks(execute=True): + self.collection.save() + mock_dispatch.assert_not_called() + mock_test.assert_not_called() + mock_prod.assert_not_called() + mock_promote.assert_not_called() + + def test_indexing_finished_triggers_nothing(self): + """INDEXING_FINISHED_ON_DEV is a Sinequa-era status: its full-text-fetch trigger is + gone (the P4 ingest replaces the fetch entirely).""" + self._save_and_assert_no_triggers(workflow_status=WorkflowStatusChoices.INDEXING_FINISHED_ON_DEV) + + def test_ready_for_curation_triggers_nothing(self): + """The Sinequa indexer-config branch on READY_FOR_CURATION is gone.""" + self._save_and_assert_no_triggers(workflow_status=WorkflowStatusChoices.READY_FOR_CURATION) + + def test_no_sinequa_methods_remain(self): + """P6: every Sinequa method is gone from Collection, and fetch_full_text from tasks.""" + from sde_collections import tasks + + for method in [ + "add_to_public_query", + "create_scraper_config", + "create_indexer_config", + "create_scraper_job", + "create_indexer_job", + "update_config_xml", + "import_metadata_from_sinequa_config", + "sinequa_configuration", + "server_url_prod", + "server_url_secret_prod", + "_write_to_github", + ]: + assert not hasattr(Collection, method), f"Collection.{method} should be deleted" + for task in ["fetch_full_text", "import_candidate_urls_from_api", "push_to_github_task"]: + assert not hasattr(tasks, task), f"tasks.{task} should be deleted" + + @patch("sde_collections.tasks.index_collection_to_test.delay") @patch("sde_collections.models.collection.Collection.promote_to_curated") - def test_curated_triggers_promotion(self, mock_promote): - """When status changes to CURATED, it should promote DeltaUrls to CuratedUrls""" + def test_curated_promotes_and_enqueues_test_indexing(self, mock_promote, mock_index_test): + """CURATED promotes DeltaUrls to CuratedUrls AND hands off to test indexing.""" self.collection.workflow_status = WorkflowStatusChoices.CURATED - self.collection.save() + with self.captureOnCommitCallbacks(execute=True): + self.collection.save() mock_promote.assert_called_once() + mock_index_test.assert_called_once_with(self.collection.id) + + @patch("sde_collections.tasks.index_collection_to_prod.delay") + def test_quality_check_statuses_enqueue_prod_indexing(self, mock_index_prod): + """QC_PERFECT / QC_MINOR hand off to prod indexing (add_to_public_query is deleted).""" + for status in [WorkflowStatusChoices.QUALITY_CHECK_PERFECT, WorkflowStatusChoices.QUALITY_CHECK_MINOR]: + collection = CollectionFactory() + collection.workflow_status = status + with self.captureOnCommitCallbacks(execute=True): + collection.save() + + mock_index_prod.assert_called_with(collection.id) + assert mock_index_prod.call_count == 2 + + @patch("sde_collections.tasks.index_collection_to_test.delay") + def test_reentrancy_guard_prevents_recursion(self, mock_index_test): + """A save() performed inside a trigger must not re-fire the dispatcher.""" + + def promote_and_save_again(): + # Simulates a trigger mutating and saving the same instance. + self.collection.save() + + with patch( + "sde_collections.models.collection.Collection.promote_to_curated", + side_effect=promote_and_save_again, + ) as mock_promote: + self.collection.workflow_status = WorkflowStatusChoices.CURATED + with self.captureOnCommitCallbacks(execute=True): + self.collection.save() - @patch("sde_collections.models.collection.Collection.add_to_public_query") - def test_quality_check_perfect_triggers_public_query(self, mock_add): - """When status changes to QUALITY_CHECK_PERFECT, it should add to public query""" - self.collection.workflow_status = WorkflowStatusChoices.QUALITY_CHECK_PERFECT - self.collection.save() + mock_promote.assert_called_once() + mock_index_test.assert_called_once() - mock_add.assert_called_once() +class TestSlackNotificationOnTransition(TestCase): + """P5: the Slack send moved out of Collection.save() into the post_save receiver.""" -class TestReindexingStatusTransitions(TestCase): def setUp(self): - # Mock the GitHubHandler to return valid XML content - self.mock_github_handler = patch("sde_collections.models.collection.GitHubHandler").start() - - self.mock_github_handler.return_value._get_file_contents.return_value.decoded_content = ( - b'\n' - b"\n" - b" false\n" - b" Sample Collection\n" - b"" - ) - - self.addCleanup(patch.stopall) - - # Create the collection with the mock applied - self.collection = CollectionFactory( - workflow_status=WorkflowStatusChoices.QUALITY_CHECK_PERFECT, - reindexing_status=ReindexingStatusChoices.REINDEXING_NOT_NEEDED, - ) + self.collection = CollectionFactory(workflow_status=WorkflowStatusChoices.RESEARCH_IN_PROGRESS) - @patch("sde_collections.tasks.fetch_full_text.delay") - def test_reindexing_finished_triggers_full_text_fetch(self, mock_fetch): - """When reindexing status changes to FINISHED, it should trigger full text fetch""" - self.collection.reindexing_status = ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV + @patch("sde_collections.tasks.dispatch_scrape_job.delay") + @patch("sde_collections.models.collection.send_slack_message") + def test_mapped_transition_sends_message_once(self, mock_send, mock_dispatch): + self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING self.collection.save() - mock_fetch.assert_called_once_with(self.collection.id, "lrm_dev") + mock_send.assert_called_once() + message = mock_send.call_args.args[0] + assert self.collection.name in message + assert "Ready for engineering" in message - @patch("sde_collections.models.collection.Collection.promote_to_curated") - def test_reindexing_curated_triggers_promotion(self, mock_promote): - """When reindexing status changes to CURATED, it should promote DeltaUrls""" - self.collection.reindexing_status = ReindexingStatusChoices.REINDEXING_CURATED - self.collection.save() - - mock_promote.assert_called_once() + def test_prod_handoff_transitions_are_mapped(self): + """The prod hand-off goes QC_* -> PRODUCTION_INDEXING -> PROD_*; the 'live on prod' + message must be keyed on the transition the poller actually makes.""" + from sde_collections.utils.slack_utils import STATUS_CHANGE_NOTIFICATIONS + for final in [WorkflowStatusChoices.PROD_PERFECT, WorkflowStatusChoices.PROD_MINOR]: + details = STATUS_CHANGE_NOTIFICATIONS[(WorkflowStatusChoices.PRODUCTION_INDEXING, final)] + assert "live on Public Prod" in details["message"] -class TestFullTextImport(TestCase): - def setUp(self): - self.collection = CollectionFactory() - self.existing_dump = DumpUrlFactory(collection=self.collection) - self.api_response = [ - {"url": "http://example.com/1", "title": "Title 1", "full_text": "Content 1"}, - {"url": "http://example.com/2", "title": "Title 2", "full_text": "Content 2"}, - ] - - @patch("sde_collections.utils.slack_utils.send_detailed_import_notification") - @patch("sde_collections.tasks.Api") - @patch("sde_collections.models.collection.GitHubHandler") - def test_full_text_import_workflow(self, MockGitHub, MockApi, MockSlackNotification): - """Test the full process of importing full text data""" - # Setup mock GitHub handler with proper XML content - mock_github = Mock() - mock_github.check_file_exists.return_value = True - mock_file_contents = Mock() - # Include all the fields that convert_template_to_plugin_indexer checks for - mock_xml = """ - - crawler2 - - - - - - - 1 - - false - - SMD_Plugins/Sinequa.Plugin.WebCrawler_Index_URLList - 3 - - - - - - - - - - true - - true - - - - - - - true - false - false - true - true - false - true - true - false - true - true - true - true - false - - - - true - no - false - - false - false - false - false - false - - - false - true - true - true - false - false - true - false - false - false - false - false - false - false - - - - true - true - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - expBackoff+headers - false - - - - - - - - - - - - false - true - - - - - false - - - false - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - - true - - true - - - true - false - - - - - - INFO - - false - - true - false - - - - false - false - false - false - true - false - - - - false - false - - - false - false - false - - - - - - - - - false - false - false - - - - true - - - - - - false - false - false - - true - - false - true - true - false - false - false - false - - - - false - false - true - false - - - true - false - - - - true - - - - - - - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - false - - - - - - - - - - false - - - - - - false - - - - - - false - - - id - doc.url1 - - false - false - """ - mock_file_contents.decoded_content = mock_xml.encode("utf-8") - mock_github._get_file_contents.return_value = mock_file_contents - MockGitHub.return_value = mock_github - - # Setup mock API - mock_api = Mock() - mock_api.get_full_texts.return_value = iter([self.api_response]) - MockApi.return_value = mock_api - - # Setup initial workflow state - self.collection.workflow_status = WorkflowStatusChoices.INDEXING_FINISHED_ON_DEV + @patch("sde_collections.models.collection.send_slack_message") + def test_unmapped_transition_sends_nothing(self, mock_send): + self.collection.workflow_status = WorkflowStatusChoices.MERGE_PENDING self.collection.save() - # Step 1: Run fetch_full_text - with patch("sde_collections.models.collection.Collection.queue_necessary_classifications") as mock_queue: - fetch_full_text(self.collection.id, "lrm_dev") - mock_queue.assert_called_once() - - # Verify old DumpUrls were cleared and new ones were also created - assert not DumpUrl.objects.filter(id=self.existing_dump.id).exists() - new_dumps = DumpUrl.objects.filter(collection=self.collection) - assert new_dumps.count() == 2 - assert {dump.url for dump in new_dumps} == {"http://example.com/1", "http://example.com/2"} + mock_send.assert_not_called() - # Step 2: Run migrate_dump_to_delta - with patch("sde_collections.models.collection.Collection.migrate_dump_to_delta") as mock_migrate: - migrate_dump_to_delta_and_handle_status_transistions(self.collection.id) - mock_migrate.assert_called_once() + @patch("sde_collections.models.collection.send_slack_message", side_effect=Exception("slack down")) + @patch("sde_collections.tasks.dispatch_scrape_job.delay") + def test_slack_failure_does_not_break_the_save(self, mock_dispatch, mock_send): + self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING + self.collection.save() # must not raise - # Verify status updates self.collection.refresh_from_db() - assert self.collection.workflow_status == WorkflowStatusChoices.READY_FOR_CURATION + assert self.collection.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING -class TestErrorHandling(TransactionTestCase): +class TestReindexingStatusTransitions(TestCase): def setUp(self): - self.collection = CollectionFactory(workflow_status=WorkflowStatusChoices.RESEARCH_IN_PROGRESS) - - @patch("sde_collections.models.collection.Collection.create_scraper_config") - @patch("sde_collections.models.collection.Collection.create_indexer_config") - def test_config_creation_failure_handling(self, mock_indexer, mock_scraper): - """Test handling of config creation failures""" - mock_scraper.side_effect = Exception("Config creation failed") - - initial_status = self.collection.workflow_status + self.collection = CollectionFactory( + workflow_status=WorkflowStatusChoices.PROD_PERFECT, + reindexing_status=ReindexingStatusChoices.REINDEXING_NOT_NEEDED, + ) - with pytest.raises(Exception): - with transaction.atomic(): - self.collection.workflow_status = WorkflowStatusChoices.READY_FOR_ENGINEERING - self.collection.save() + def test_reindexing_finished_triggers_nothing(self): + """REINDEXING_FINISHED_ON_DEV must trigger nothing: the P4 ingest sets this status + itself — a trigger here would double-fire.""" + with ( + patch("sde_collections.tasks.dispatch_scrape_job.delay") as mock_dispatch, + patch("sde_collections.models.collection.Collection.promote_to_curated") as mock_promote, + ): + self.collection.reindexing_status = ReindexingStatusChoices.REINDEXING_FINISHED_ON_DEV + self.collection.save() - # Verify status wasn't changed on error - self.collection.refresh_from_db() - assert self.collection.workflow_status == initial_status + mock_dispatch.assert_not_called() + mock_promote.assert_not_called() - @patch("sde_collections.tasks.Api") - def test_full_text_fetch_failure_handling(self, MockApi): - """Test handling of full text fetch failures""" - mock_api = Mock() - mock_api.get_full_texts.side_effect = Exception("API error") - MockApi.return_value = mock_api + @patch("sde_collections.models.collection.Collection.promote_to_curated") + def test_reindexing_curated_triggers_promotion(self, mock_promote): + """When reindexing status changes to CURATED, it should promote DeltaUrls""" + self.collection.reindexing_status = ReindexingStatusChoices.REINDEXING_CURATED + self.collection.save() - initial_status = self.collection.workflow_status + mock_promote.assert_called_once() - with pytest.raises(Exception): - fetch_full_text(self.collection.id, "lrm_dev") - # Verify status wasn't changed on error - self.collection.refresh_from_db() - assert self.collection.workflow_status == initial_status +NEW_PIPELINE_STATUSES = [ + WorkflowStatusChoices.SCRAPING_SUCCESSFUL, + WorkflowStatusChoices.TEST_INDEXING, + WorkflowStatusChoices.SCRAPING_FAILED, + WorkflowStatusChoices.INDEXING_FAILED_ON_TEST, + WorkflowStatusChoices.INDEXING_FAILED_ON_PROD, + WorkflowStatusChoices.PRODUCTION_INDEXING, +] + +VALID_BUTTON_COLORS = { + "btn-light", + "btn-danger", + "btn-warning", + "btn-info", + "btn-success", + "btn-primary", + "btn-secondary", +} + + +@pytest.mark.parametrize("status", list(WorkflowStatusChoices)) +def test_collection_button_color_resolves_for_every_status(status): + """Regression guard: an unmapped status used to raise KeyError and break the + collection list and detail pages. Every enum member must resolve a colour.""" + collection = Collection(workflow_status=status) + assert collection.workflow_status_button_color in VALID_BUTTON_COLORS + + +@pytest.mark.parametrize("status", list(WorkflowStatusChoices)) +def test_workflow_history_button_color_resolves_for_every_status(status): + history = WorkflowHistory(workflow_status=status) + assert history.workflow_status_button_color in VALID_BUTTON_COLORS + + +def test_unmapped_status_falls_back_to_default_color(): + assert Collection(workflow_status=999).workflow_status_button_color == "btn-light" + assert WorkflowHistory(workflow_status=999).workflow_status_button_color == "btn-light" + + +@pytest.mark.parametrize( + "status", + [ + WorkflowStatusChoices.SCRAPING_FAILED, + WorkflowStatusChoices.INDEXING_FAILED_ON_TEST, + WorkflowStatusChoices.INDEXING_FAILED_ON_PROD, + ], +) +def test_failure_statuses_do_not_render_neutral(status): + assert Collection(workflow_status=status).workflow_status_button_color == "btn-danger" + assert WorkflowHistory(workflow_status=status).workflow_status_button_color == "btn-danger" + + +@pytest.mark.django_db +@pytest.mark.parametrize("status", NEW_PIPELINE_STATUSES) +def test_setting_new_status_writes_workflow_history(status): + collection = CollectionFactory() + collection.workflow_status = status + collection.save() + assert WorkflowHistory.objects.filter(collection=collection, workflow_status=status).exists() diff --git a/sde_collections/urls.py b/sde_collections/urls.py index 9ee77759..ffdc1b39 100644 --- a/sde_collections/urls.py +++ b/sde_collections/urls.py @@ -23,16 +23,6 @@ path("", view=views.CollectionListView.as_view(), name="list"), path("sde-dashboard/", view=views.SdeDashboardView.as_view(), name="dashboard"), path("/", view=views.CollectionDetailView.as_view(), name="detail"), - path( - "api/collections/push_to_github/", - views.PushToGithubView.as_view(), - name="push-to-github", - ), - path( - "api/indexing_instructions/", - views.IndexingInstructionsView.as_view(), - name="indexing_instructions", - ), path("api/assign-division//", views.DeltaURLViewSet.as_view({"post": "update_division"})), path( "delete-required-url/", @@ -44,11 +34,6 @@ view=views.DeltaURLsListView.as_view(), name="delta_urls", ), - path( - "consolidate/", - view=views.WebappGitHubConsolidationView.as_view(), - name="consolidate_db_and_github_configs", - ), # List all DeltaURL instances: /delta-urls/ # Retrieve a specific DeltaURL instance: /delta-urls/{id}/ # Create a new DeltaURL instance: /delta-urls/ diff --git a/sde_collections/utils/aws.py b/sde_collections/utils/aws.py new file mode 100644 index 00000000..7e793f4b --- /dev/null +++ b/sde_collections/utils/aws.py @@ -0,0 +1,19 @@ +import boto3 +from django.conf import settings + + +def get_boto3_session(): + """Default credential chain (instance role in AWS); explicit SDE keys only if set (local dev). + + Deliberately does NOT read the DJANGO_AWS_* / AWS_ACCESS_KEY_ID settings — those are the + django-storages static-assets credentials, a different scope (and absent under test settings). + """ + if settings.SDE_AWS_ACCESS_KEY_ID and settings.SDE_AWS_SECRET_ACCESS_KEY: + return boto3.Session( + aws_access_key_id=settings.SDE_AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.SDE_AWS_SECRET_ACCESS_KEY, + # Set only for temporary (SSO / STS) credentials; blank for long-lived IAM keys. + aws_session_token=settings.SDE_AWS_SESSION_TOKEN or None, + region_name=settings.AWS_REGION, + ) + return boto3.Session(region_name=settings.AWS_REGION) diff --git a/sde_collections/utils/bulk_github_push.py b/sde_collections/utils/bulk_github_push.py deleted file mode 100644 index 420992a4..00000000 --- a/sde_collections/utils/bulk_github_push.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Sometimes it is necessary to programatically push many collections at once to github. -This code will search for collections matching a certain criteria (curated, pr created), -and push their changes to Github -""" - -from sde_collections.models.collection import Collection -from sde_collections.models.collection_choice_fields import CurationStatusChoices -from sde_collections.utils.github_helper import GitHubHandler - -FINISHED_STATUSES = [ - CurationStatusChoices.CURATED, - CurationStatusChoices.GITHUB_PR_CREATED, -] - - -# currently, the existing automated github branch needs to be deleted -def bulk_push(statuses_to_push=FINISHED_STATUSES): - collections = Collection.objects.filter(curation_status__in=statuses_to_push).exclude(name__icontains="fake") - - gh = GitHubHandler(collections) - gh.push_to_github() diff --git a/sde_collections/utils/generate_deployment_message.py b/sde_collections/utils/generate_deployment_message.py index 19ecd2ec..eb91ea15 100644 --- a/sde_collections/utils/generate_deployment_message.py +++ b/sde_collections/utils/generate_deployment_message.py @@ -12,7 +12,7 @@ def generate_deployment_message(collection_config_folders): to our production environment as part of our latest deployment! :tada:\n Collections Now Live in Prod:\n""" - message_middle = "\n\n".join([f"- {collection.name} | {collection.server_url_prod}" for collection in collections]) + message_middle = "\n\n".join([f"- {collection.name}" for collection in collections]) message_end = """ If you find something needs changing, please let us know. diff --git a/sde_collections/utils/github_helper.py b/sde_collections/utils/github_helper.py deleted file mode 100644 index 62be7088..00000000 --- a/sde_collections/utils/github_helper.py +++ /dev/null @@ -1,227 +0,0 @@ -from django.conf import settings -from github import Github -from github.GithubException import GithubException, UnknownObjectException - -from config_generation.db_to_xml import XmlEditor - -from ..models.collection_choice_fields import CurationStatusChoices - - -class GitHubHandler: - def __init__(self, collections=None, *args, **kwargs): - self.github_token = settings.GITHUB_ACCESS_TOKEN - self.g = Github(self.github_token) - self.repo = self.g.get_repo(f"{settings.SINEQUA_CONFIGS_GITHUB_REPO}") - self.master_branch = settings.SINEQUA_CONFIGS_REPO_MASTER_BRANCH - self.dev_branch = settings.SINEQUA_CONFIGS_REPO_DEV_BRANCH - self.webapp_pr_branch = settings.SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH - self.collections = collections - # if we still need operations performed on a collection list - # we should refactor, as this functionality should not be a - # part of the base GithubHandler class. - # maybe it should just be passed to an individual method - - def _get_file_contents(self, file_path, strict=True): - """ - try to get file contents, first from the pr branch and then from the dev branch - if strict is True, raise an exception if the file is not found - """ - - try: - contents = self.repo.get_contents(file_path, ref=self.webapp_pr_branch) - except UnknownObjectException: - try: - contents = self.repo.get_contents(file_path, ref=self.dev_branch) - except UnknownObjectException: - if strict: - raise Exception( - f"File {file_path} not found on {self.dev_branch} or {self.webapp_pr_branch} branches" - ) - return None - - return contents - - def check_file_exists(self, file_path): - """ - Check if file exists on GitHub - """ - - if self._get_file_contents(file_path, strict=False) is None: - return False - else: - return True - - def create_file(self, file_path, file_string, branch=None): - """ - Create file contents on GitHub - if no branch is provided, it will default to the webapp_pr_branch - """ - - if not branch: - branch = self.webapp_pr_branch - - if self.check_file_exists(file_path): - raise Exception(f"File {file_path} already exists on GitHub") - - COMMIT_MESSAGE = f"Webapp: Create {file_path}" - - self.repo.create_file( - file_path, - COMMIT_MESSAGE, - file_string, - branch=branch, - ) - - def create_or_update_file(self, file_path, file_string, branch=None): - """ - Update file contents on GitHub - if no branch is provided, it will default to the webapp_pr_branch - """ - - if not branch: - branch = self.webapp_pr_branch - - if self.check_file_exists(file_path): - contents = self._get_file_contents(file_path) - COMMIT_MESSAGE = f"Webapp: Update {file_path}" - - self.repo.update_file( - contents.path, - COMMIT_MESSAGE, - file_string, - contents.sha, - branch=branch, - ) - else: - self.create_file(file_path, file_string, branch) - - def update_config_with_current_rules(self, collection): - """ - DEPRECATED? - this runs the update_config_xml method from the collection model - which adds the latest rules to the xml file - """ - contents = self._get_file_contents(collection) - FILE_CONTENTS = contents.decoded_content.decode("utf-8") - updated_xml = collection.update_config_xml(FILE_CONTENTS) - - COMMIT_MESSAGE = f"Webapp: Update {collection.name}" - - self.repo.update_file( - contents.path, - COMMIT_MESSAGE, - updated_xml, - contents.sha, - branch=self.github_branch, - ) - - def branch_exists(self, branch_name: str) -> bool: - try: - self.repo.get_branch(branch=branch_name) - return True - except GithubException: - return False - - def create_branch(self, branch_name: str): - # Get the SHA of the commit you want to branch from (basically the Dev branch) - base_sha = self.repo.get_branch(self.dev_branch).commit.sha - # Create the new branch - self.repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=base_sha) - - def create_pull_request(self) -> None: - title = "Webapp: Update config files" - body = "\n".join(self.collections.values_list("name", flat=True)) - try: - self.repo.create_pull( - title=title, - body=body, - base=self.dev_branch, - head=self.github_branch, - ) - except GithubException: # PR exists - print("PR exists") - - def push_to_github(self) -> None: - if not self.branch_exists(self.github_branch): - self.create_branch(self.github_branch) - for collection in self.collections: - print(f"Pushing {collection.name} to GitHub.") - self.update_config_with_current_rules(collection) - collection.curation_status = CurationStatusChoices.GITHUB_PR_CREATED - collection.save() - self.create_pull_request() - - def fetch_metadata(self): - metadata = {} - for collection in self.collections: - contents = self._get_file_contents(collection) - - if not contents: - continue - - FILE_CONTENTS = contents.decoded_content.decode("utf-8") - collection_xml = XmlEditor(FILE_CONTENTS) - - tree_root = collection_xml.fetch_treeroot() - document_type = collection_xml.fetch_document_type() - - metadata[collection.config_folder] = { - "tree_root": tree_root, - "document_type": document_type, - } - return metadata - - def _get_config_folder(self, collection_folder): - return collection_folder.removeprefix("sources/SDE/") - - def _get_list_of_collections(self): - BASE_PATH = "sources/SDE" - collections = self.repo.get_contents(BASE_PATH, ref=self.dev_branch) - collection_folders = [ - collection.path - for collection in collections - if ".xml" not in collection.path # to prevent source.xml from being included - ] - return collection_folders - - def _get_contents_from_path(self, path): - # we don't need to check if the file exists because we already did that in _get_list_of_collections - contents = self.repo.get_contents(path, ref=self.dev_branch) - FILE_CONTENTS = contents.decoded_content.decode("utf-8") - collection_xml = XmlEditor(FILE_CONTENTS) - return collection_xml - - def get_collections_from_github(self, config_folders=[]): - # get a list of folders in sources/SDE/ from the dev branch on github - collection_folders = self._get_list_of_collections() - - # create a dict of all collections and their metadata - collection_list = [] - - # for each folder in the list, get the metadata: config_folder, name, url, division, tree_root, document_type - for collection_folder in collection_folders: - config_folder = self._get_config_folder(collection_folder) - collection_xml_file_path = self._get_config_file_path(config_folder) - collection_xml = self._get_contents_from_path(collection_xml_file_path) - - division, name = collection_xml.fetch_division_name() - - if not division or not name: - print(f"Skipping {config_folder} because it has no division or name") - continue - - collection_dict = { - "config_folder": config_folder, - "name": name, - "url": collection_xml.fetch_url(), - "division": division, - "document_type": collection_xml.fetch_document_type(), - "connector": collection_xml.fetch_connector(), - } - collection_list.append(collection_dict) - - # return the list of collections and their metadata - return collection_list - - def sync_rules_with_github(self, config_folders=[]): - pass diff --git a/sde_collections/utils/health_check.py b/sde_collections/utils/health_check.py deleted file mode 100644 index 0e09bd87..00000000 --- a/sde_collections/utils/health_check.py +++ /dev/null @@ -1,273 +0,0 @@ -import json -import re - -import boto3 -import botocore -from django.conf import settings - -from sde_collections.models.candidate_url import CandidateURL -from sde_collections.models.collection import ( - Collection, - CurationStatusChoices, - WorkflowStatusChoices, -) -from sde_collections.models.collection_choice_fields import ( - ConnectorChoices, - Divisions, - DocumentTypes, -) -from sde_collections.models.pattern import ExcludePattern, TitlePattern -from sde_collections.tasks import ( - _get_data_to_import, - pull_latest_collection_metadata_from_github, -) - - -def health_check(collection, server_name: str = "production") -> dict: - """ - This method checks whether the rules defined in webapp are properly - synced with Sinequa or not. - - Checks for Title Patterns, Exclude Patterns and Document Type Patterns. - """ - health_check_report = [] - - # get candidate URLs from sinequa - candidate_urls_sinequa = _fetch_candidate_urls(collection, server_name) - - # check for title patterns - title_pattern_report = _health_check_title_pattern(collection, candidate_urls_sinequa) - health_check_report.extend(title_pattern_report) - - # check for exclude patterns - exclude_pattern_report = _health_check_exclude_pattern(collection, candidate_urls_sinequa) - health_check_report.extend(exclude_pattern_report) - - return health_check_report - - -def _fetch_candidate_urls(collection, server_name): - # TODO: should we make it available throughout the project?? - candidate_urls_remote = _get_data_to_import(collection, server_name) - - candidate_urls_sinequa = {} - for candidate_url in candidate_urls_remote: - url = candidate_url["fields"]["url"] - candidate_urls_sinequa[url] = CandidateURL(url=url, scraped_title=candidate_url["fields"]["scraped_title"]) - return candidate_urls_sinequa - - -def _health_check_title_pattern(collection, candidate_urls_sinequa): - collection_id = collection.pk - collection_name = collection.name - collection_config_folder = collection.config_folder - curation_status = collection.curation_status - workflow_status = collection.workflow_status - - title_pattern_report = [] - - # now get Title Patterns in indexer db - title_patterns_local = TitlePattern.objects.all().filter(collection_id=collection_id) - - # check if title patterns are porperly reflected in sinequa's response - for title_pattern in title_patterns_local: - pattern = title_pattern.title_pattern - matched_urls = title_pattern.matched_urls() - - # now check to see if the matched_urls and candidate_urls_sinequa are similar or not - for matched_url in matched_urls: - url = matched_url.url - if url in candidate_urls_sinequa: - matched_title = matched_url.scraped_title - sinequa_title = candidate_urls_sinequa[url].scraped_title - - if _resolve_title_pattern(pattern, sinequa_title) is None: - report = { - "id": collection_id, - "collection_name": collection_name, - "config_folder": collection_config_folder, - "curation_status": CurationStatusChoices.get_status_string(curation_status), - "workflow_status": WorkflowStatusChoices.get_status_string(workflow_status), - "pattern_name": "Title Pattern", - "pattern": pattern, - "scraped_title": matched_title, - "non_compliant_url": matched_url, - } - title_pattern_report.append(report) - - return title_pattern_report - - -def _health_check_exclude_pattern(collection, candidate_urls_sinequa): - collection_id = collection.pk - collection_name = collection.name - collection_config_folder = collection.config_folder - curation_status = collection.curation_status - workflow_status = collection.workflow_status - - exclude_pattern_report = [] - - # Perform exclude pattern check here - exclude_patterns_local = ExcludePattern.objects.all().filter(collection_id=collection_id) - - def create_exclude_pattern_report(match_pattern, url): - return { - "id": collection_id, - "collection_name": collection_name, - "config_folder": collection_config_folder, - "curation_status": CurationStatusChoices.get_status_string(curation_status), - "workflow_status": WorkflowStatusChoices.get_status_string(workflow_status), - "pattern_name": "Exclude Pattern", - "pattern": match_pattern, - "non_compliant_url": url, - } - - for exclude_pattern in exclude_patterns_local: - match_pattern = exclude_pattern.match_pattern - - # check with http:// - if match_pattern.find("http://") == -1: - url = f"http://{match_pattern}" # noqa: E231 - if url in candidate_urls_sinequa: - exclude_pattern_report.append(create_exclude_pattern_report(match_pattern, url)) - - if match_pattern.find("https://") == -1: - url = f"https://{match_pattern}" # noqa: E231 - if url in candidate_urls_sinequa: - exclude_pattern_report.append(create_exclude_pattern_report(match_pattern, url)) - else: - url = match_pattern # assuming it has either https or http - if url in candidate_urls_sinequa: - exclude_pattern_report.append(create_exclude_pattern_report(match_pattern, url)) - - return exclude_pattern_report - - -def _resolve_title_pattern(pattern, title): - """ - Given a pattern check whether it is able to capture the title or not. - - E.g.: GCN {title} - should capture : - -> GCN - Notices - -> GCN - News - """ - pattern_with_whitespace = pattern.replace(" ", r"\s*-?\s*") - - parentheis_pattern = r"\{[^\}]+\}" - multi_pattern = r"\/\/\*([^\/]*)\/a" - - def replace_parentheis_with_anything(match): - return r"\S+" - - regex_pattern_parenthesis = re.sub(parentheis_pattern, replace_parentheis_with_anything, pattern_with_whitespace) - regex_pattern = re.sub(multi_pattern, replace_parentheis_with_anything, regex_pattern_parenthesis) - return re.match(regex_pattern, title) - - -def parse_int_values(github_value, db_value, field): - """ - This method parses the integer values to their corresponding labels. - """ - field_dict = { - "division": Divisions, - "document_type": DocumentTypes, - "connector": ConnectorChoices, - } - try: - github_value = field_dict[field](github_value).label - except ValueError: - github_value = None - - try: - db_value = field_dict[field](db_value).label - except ValueError: - db_value = None - - return github_value, db_value - - -def generate_db_github_metadata_differences(reindex_configs_from_github=False): - report = [] - if reindex_configs_from_github: - pull_latest_collection_metadata_from_github.delay() - return report - - # for each folder in github get the metadata from the default.xml file - FILENAME = "github_collections.json" - - s3 = boto3.resource( - "s3", - region_name="us-east-1", - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, - ) - - try: - s3.Object(settings.AWS_STORAGE_BUCKET_NAME, FILENAME).load() - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "404": - pull_latest_collection_metadata_from_github.delay() - return report - else: - raise - else: - collections = json.load(s3.Object(settings.AWS_STORAGE_BUCKET_NAME, FILENAME).get()["Body"]) - - fields = { - "config_folder", - "name", - "url", - "division", - "document_type", - "connector", - } - - # also fetch same metadata from the database - for collection in collections: - # fix division to be the same as in the database - collection["division"] = Divisions.lookup_by_text(collection["division"]) - config_folder = collection["config_folder"] - try: - db_collection = Collection.objects.get(config_folder=config_folder) - except Collection.DoesNotExist: - report.append( - { - "config_folder": config_folder, - "field": "config_folder", - "github_value": config_folder, - "db_value": "DoesNotExist", - } - ) - continue - - # check if there is any difference in the metadata - # if there is a difference then add it to the report - int_fields = {"division", "document_type", "connector"} - for field in fields: - if collection[field] != getattr(db_collection, field): - db_value = getattr(db_collection, field) - if field in int_fields: - collection[field], db_value = parse_int_values(collection[field], db_value, field) - - report.append( - { - "config_folder": config_folder, - "field": field, - "github_value": collection[field], - "db_value": db_value, - } - ) - - # check if there are any collections in the database that are not in github - for db_collection in Collection.objects.exclude(config_folder__in=[c["config_folder"] for c in collections]): - report.append( - { - "config_folder": db_collection.config_folder, - "field": "config_folder", - "github_value": "DoesNotExist", - "db_value": db_collection.config_folder, - } - ) - - return report diff --git a/sde_collections/utils/slack_utils.py b/sde_collections/utils/slack_utils.py index 292ef897..4cc40a91 100644 --- a/sde_collections/utils/slack_utils.py +++ b/sde_collections/utils/slack_utils.py @@ -1,8 +1,12 @@ +import logging + import requests from django.conf import settings from ..models.collection_choice_fields import WorkflowStatusChoices +logger = logging.getLogger(__name__) + SLACK_ID_MAPPING = { "Shravan Vishwanathan": "<@U056B4HMGEP>", "Advait Yogaonkar": "<@U06L5SKQ5QA>", @@ -46,6 +50,53 @@ "message": "{name} is now live on Public Prod! Congrats team! :sparkles:", "mention_users": ["channel"], }, + # The prod hand-off passes through PRODUCTION_INDEXING, so the transition the poller + # actually makes is PRODUCTION_INDEXING -> PROD_*; keep the QC_* -> PROD_* pairs above + # for the manual/legacy path. + (WorkflowStatusChoices.PRODUCTION_INDEXING, WorkflowStatusChoices.PROD_PERFECT): { + "message": "{name} is now live on Public Prod! Congrats team! :sparkles:", + "mention_users": ["channel"], + }, + (WorkflowStatusChoices.PRODUCTION_INDEXING, WorkflowStatusChoices.PROD_MINOR): { + "message": "{name} is now live on Public Prod! Congrats team! :sparkles:", + "mention_users": ["channel"], + }, + # --- SDE curation pipeline (crawl4ai scraper + web indexing) --- + # Detailed scrape counts are posted separately via send_detailed_import_notification on ingest. + (WorkflowStatusChoices.READY_FOR_ENGINEERING, WorkflowStatusChoices.SCRAPING_SUCCESSFUL): { + "message": "Scraping of {name} finished successfully. Ingest and delta migration underway! :white_check_mark:", + }, + (WorkflowStatusChoices.ENGINEERING_IN_PROGRESS, WorkflowStatusChoices.SCRAPING_SUCCESSFUL): { + "message": "Scraping of {name} finished successfully. Ingest and delta migration underway! :white_check_mark:", + }, + (WorkflowStatusChoices.READY_FOR_ENGINEERING, WorkflowStatusChoices.SCRAPING_FAILED): { + "message": "Alert: Scraping of {name} has failed! :warning:", + "mention_users": ["Shravan Vishwanathan", "Advait Yogaonkar"], + }, + (WorkflowStatusChoices.ENGINEERING_IN_PROGRESS, WorkflowStatusChoices.SCRAPING_FAILED): { + "message": "Alert: Scraping of {name} has failed! :warning:", + "mention_users": ["Shravan Vishwanathan", "Advait Yogaonkar"], + }, + (WorkflowStatusChoices.CURATED, WorkflowStatusChoices.INDEXING_FAILED_ON_TEST): { + "message": "Alert: Indexing of {name} on Test has failed! :warning:", + }, + (WorkflowStatusChoices.TEST_INDEXING, WorkflowStatusChoices.INDEXING_FAILED_ON_TEST): { + "message": "Alert: Indexing of {name} on Test has failed! :warning:", + }, + # INDEXING_FAILED_ON_PROD can be reached from any of the prod hand-off statuses; + # the lookup is exact (old, new) pairs, so each realistic predecessor is listed. + (WorkflowStatusChoices.PRODUCTION_INDEXING, WorkflowStatusChoices.INDEXING_FAILED_ON_PROD): { + "message": "Alert: Indexing of {name} on Prod has failed! :warning:", + "mention_users": ["Shravan Vishwanathan", "Advait Yogaonkar"], + }, + (WorkflowStatusChoices.QUALITY_CHECK_PERFECT, WorkflowStatusChoices.INDEXING_FAILED_ON_PROD): { + "message": "Alert: Indexing of {name} on Prod has failed! :warning:", + "mention_users": ["Shravan Vishwanathan", "Advait Yogaonkar"], + }, + (WorkflowStatusChoices.QUALITY_CHECK_MINOR, WorkflowStatusChoices.INDEXING_FAILED_ON_PROD): { + "message": "Alert: Indexing of {name} on Prod has failed! :warning:", + "mention_users": ["Shravan Vishwanathan", "Advait Yogaonkar"], + }, } @@ -75,7 +126,43 @@ def send_detailed_import_notification( payload = {"text": message} response = requests.post(webhook_url, json=payload) if response.status_code != 200: - print(f"Error sending Slack message: {response.text}") + logger.warning("Error sending Slack message: %s", response.text) + + +def send_indexing_validation_report(collection_name, run_id, validation): + """Post the indexer-produced validation.json (WORKFLOW.md steps 22-25 QC report) to + the curation channel so the curator can set the QC status from it.""" + if validation is None: + message = ( + f"Test indexing of '{collection_name}' succeeded (run {run_id}), " f"but no validation report was found." + ) + else: + missing = validation.get("titles_missing_in_index") or [] + extra = validation.get("titles_only_in_index") or [] + message = ( + f"Test indexing of '{collection_name}' succeeded (run {run_id}). QC report:\n" + f"Expected documents: {validation.get('expected_count')}\n" + f"Indexed documents: {validation.get('indexed_count')}\n" + f"Counts match: {validation.get('count_matches')}\n" + f"Title match rate: {validation.get('title_match_rate')}\n" + f"Titles missing from index: {len(missing)}\n" + f"Titles only in index: {len(extra)}\n" + f"Please review and set the QC status." + ) + send_slack_message(message) + + +def notify_status_change(collection_name, collection_id, old_status, new_status): + """Post the mapped message for a transition that was written with a queryset + .update() (which bypasses post_save and therefore the model-level notifier). + No-op for unmapped transitions; never raises.""" + details = STATUS_CHANGE_NOTIFICATIONS.get((old_status, new_status)) + if details is None: + return + try: + send_slack_message(format_slack_message(collection_name, details, collection_id)) + except Exception as e: + logger.warning("Error sending Slack message for %s: %s", collection_name, e) def send_slack_message(message): diff --git a/sde_collections/views.py b/sde_collections/views.py index eba0b5e9..7b496007 100644 --- a/sde_collections/views.py +++ b/sde_collections/views.py @@ -1,29 +1,23 @@ import re -from django.contrib import messages from django.contrib.auth import get_user_model from django.contrib.auth.mixins import LoginRequiredMixin from django.db import models from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils import timezone -from django.views.generic import TemplateView, View +from django.views.generic import View from django.views.generic.detail import DetailView from django.views.generic.edit import DeleteView from django.views.generic.list import ListView from rest_framework import generics, status, viewsets -from rest_framework.exceptions import ValidationError from rest_framework.generics import ListAPIView from rest_framework.response import Response -from rest_framework.views import APIView from .forms import CollectionGithubIssueForm, CommentsForm, RequiredUrlForm from .models.collection import Collection, Comments, RequiredUrls, WorkflowHistory from .models.collection_choice_fields import ( - ConnectorChoices, CurationStatusChoices, - Divisions, - DocumentTypes, ReindexingStatusChoices, WorkflowStatusChoices, ) @@ -51,8 +45,6 @@ IncludePatternSerializer, TitlePatternSerializer, ) -from .tasks import push_to_github_task -from .utils.health_check import generate_db_github_metadata_differences User = get_user_model() @@ -500,119 +492,6 @@ class CollectionReadViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = CollectionReadSerializer -class PushToGithubView(APIView): - def post(self, request): - collection_ids = request.POST.getlist("collection_ids[]", []) - if len(collection_ids) == 0: - return Response("collection_ids can't be empty.", status=status.HTTP_400_BAD_REQUEST) - - push_to_github_task.delay(collection_ids) - - return Response( - {"Success": "Started pushing collections to github"}, - status=status.HTTP_200_OK, - ) - - -class IndexingInstructionsView(APIView): - """ - Serves the name of the first curated collection to be indexed and updates collection workflow status - """ - - def get(self, request): - curated_collections = Collection.objects.filter(workflow_status=WorkflowStatusChoices.CURATED) - - job_name = "" - if curated_collections.exists(): - collection = curated_collections.first() - job_name = f"collection.indexer.{collection.config_folder}.xml" - - return Response( - { - "job_name": job_name, - }, - status=status.HTTP_200_OK, - ) - - def post(self, request): - config_folder = request.data.get("collection") - indexing_status = request.data.get("status") - - if not config_folder or not indexing_status: - raise ValidationError({"error": "Config folder name and indexing status are required."}) - - collection = get_object_or_404(Collection, config_folder=config_folder) - - if indexing_status == "STARTED_INDEXING" and collection.workflow_status == WorkflowStatusChoices.CURATED: - collection.workflow_status = WorkflowStatusChoices.SECRET_DEPLOYMENT_STARTED - collection.save() - - return Response( - {"message": f"Status for collection '{collection.name}' updated to Secret Deployment Started."}, - status=status.HTTP_200_OK, - ) - elif ( - indexing_status == "FINISHED_INDEXING" - and collection.workflow_status == WorkflowStatusChoices.SECRET_DEPLOYMENT_STARTED - ): - collection.workflow_status = WorkflowStatusChoices.READY_FOR_LRM_QUALITY_CHECK - collection.save() - - return Response( - {"message": f"Status for collection '{collection.name}' updated to Ready For LRM Quality Check."}, - status=status.HTTP_200_OK, - ) - else: - return Response({"error": "Invalid indexing or workflow status."}, status=status.HTTP_400_BAD_REQUEST) - - -class WebappGitHubConsolidationView(LoginRequiredMixin, TemplateView): - """ - Display a list of collections in the system - """ - - template_name = "sde_collections/consolidate_db_and_github_configs.html" - - def get(self, request, *args, **kwargs): - if not request.GET.get("reindex") == "true": - self.data = generate_db_github_metadata_differences() - else: - # this needs to be a celery task eventually - self.data = generate_db_github_metadata_differences(reindex_configs_from_github=True) - - return super().get(request, *args, **kwargs) - - def post(self, request, *args, **kwargs): - config_folder = self.request.POST.get("config_folder") - field = self.request.POST.get("field") - new_value = self.request.POST.get("github_value") - - if new_value and new_value != "None": - new_value = new_value.strip() - if field == "division": - new_value = Divisions.lookup_by_text(new_value) - elif field == "document_type": - new_value = DocumentTypes.lookup_by_text(new_value) - elif field == "connector": - new_value = ConnectorChoices.lookup_by_text(new_value) - - Collection.objects.filter(config_folder=config_folder).update(**{field: new_value}) - messages.success(request, f"Successfully updated {field} of {config_folder}.") - else: - messages.error( - request, - f"Can't update empty value from GitHub: {field} of {config_folder}.", - ) - - return redirect("sde_collections:consolidate_db_and_github_configs") - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context["differences"] = self.data - - return context - - class ResolvedTitleListView(ListView): model = DeltaResolvedTitle context_object_name = "resolved_titles" diff --git a/sde_collections/xml_templates/new_collection_template.xml b/sde_collections/xml_templates/new_collection_template.xml deleted file mode 100644 index 0ee71927..00000000 --- a/sde_collections/xml_templates/new_collection_template.xml +++ /dev/null @@ -1,292 +0,0 @@ - - - A new collection template - - crawler2 - - - - - - false - - - - - - - - - - false - false - false - - - true - - - - - - false - false - false - 0 - - - - - false - - true - false - - - false - false - false - false - true - false - - - - true - false - - false - false - false - - - - - - - - - - - - - - - - false - true - true - false - false - false - - - - false - false - true - false - - _Advanced - true - false - - - - true - - - - - - - false - false - false - false - false - false - false - false - false - false - false - true - true - false - false - false - false - true - false - - false - false - false - - - - - - - - - false - - - - - - false - - - - - - false - - 3 - - - - true - true - true - 100 - 100000 - 100000 - 10 - -1 - -1 - true - false - false - false - false - false - true - true - false - true - true - true - true - false - 1 - 0 ms - true - no - false - - false - false - true - false - false - - - false - true - true - true - false - true - true - false - false - false - false - false - false - false - - - - true - true - - true - false - - - - false - - false - true - false - - true - - - - - - false - false - true - - - - - - - - - - - false - true - - - - - false - - - - - - - - - true - true - - - false - - - - - - - - eu-west-1 - - true - - true - - 80 - true - false - - - - - false - false - - 1 - - url_to_scrape - *.rtf;*.jy;*.xml;*.ico;*.gz;*.act - - id - doc.url1 - - - - - diff --git a/sde_indexing_helper/static/js/collection_list.js b/sde_indexing_helper/static/js/collection_list.js index 7eb5c7bf..a29536b8 100644 --- a/sde_indexing_helper/static/js/collection_list.js +++ b/sde_indexing_helper/static/js/collection_list.js @@ -331,6 +331,16 @@ function handleWorkflowStatusSelect() { 14: "btn-primary", 15: "btn-info", 16: "btn-secondary", + 17: "btn-light", + 18: "btn-success", + 19: "btn-warning", + 20: "btn-info", + 21: "btn-success", + 22: "btn-light", + 23: "btn-danger", + 24: "btn-danger", + 25: "btn-danger", + 26: "btn-light", }; $possible_buttons = $("body").find( diff --git a/sde_indexing_helper/static/js/consolidate_db_and_github_configs.js b/sde_indexing_helper/static/js/consolidate_db_and_github_configs.js deleted file mode 100644 index ca9c3771..00000000 --- a/sde_indexing_helper/static/js/consolidate_db_and_github_configs.js +++ /dev/null @@ -1,9 +0,0 @@ -let table = $('#consolidation_table').DataTable({ - "paging": false, - "stateSave": true, - "dom": 'Pfritip', - searchPanes: { - viewTotal: true, - columns: [1] - } -}); diff --git a/sde_indexing_helper/static/js/project.js b/sde_indexing_helper/static/js/project.js index 620e301a..6934961c 100644 --- a/sde_indexing_helper/static/js/project.js +++ b/sde_indexing_helper/static/js/project.js @@ -5,7 +5,7 @@ "Name": "The designated name of the collection.", "URL": "The primary URL of the collection from which the scraping process begins.", "Division": "The specific division to which the collection belongs. It can be one of: Astrophysics, Heliophysics, Biological and Physical Sciences, Earth Science, or Planetary Science.", - "Candidate URLs" : "The URLs crawled from the base URL by Sinequa. These are curated and sent for indexing.", + "Candidate URLs" : "The URLs crawled from the base URL by the scraper. These are curated and sent for indexing.", "Workflow Status": "The current stage of the collection within the workflow.", "Curator": "The individual responsible for curating this collection.", "Connector Type": "Indicates whether the connector is a web crawler or API-based." diff --git a/sde_indexing_helper/templates/sde_collections/collection_detail.html b/sde_indexing_helper/templates/sde_collections/collection_detail.html index 24be2ba0..c6a1011e 100644 --- a/sde_indexing_helper/templates/sde_collections/collection_detail.html +++ b/sde_indexing_helper/templates/sde_collections/collection_detail.html @@ -30,10 +30,6 @@

{{ colle
{{ colle Notes {{ collection.notes }} - - Sinequa Configuration - - {{collection.sinequa_configuration }} - - GitHub Issue Link diff --git a/sde_indexing_helper/templates/sde_collections/consolidate_db_and_github_configs.html b/sde_indexing_helper/templates/sde_collections/consolidate_db_and_github_configs.html deleted file mode 100644 index b3d2f8ba..00000000 --- a/sde_indexing_helper/templates/sde_collections/consolidate_db_and_github_configs.html +++ /dev/null @@ -1,58 +0,0 @@ -{% extends "layouts/base.html" %} -{% load static %} -{% load i18n %} -{% block title %}Consolidation between webapp and GitHub{% endblock %} -{% block stylesheets %} - {{ block.super }} - -{% endblock stylesheets %} -{% block content %} - {% csrf_token %} -
-

Collection metadata differences between Webapp and GitHub

- - - - - - - - - - - - - {% for difference in differences %} - - - - - - - - {% endfor %} - -
CollectionFieldWebapp ValueGitHub ValueActions
{{ difference.config_folder }}{{ difference.field }}{{ difference.db_value }}{{ difference.github_value }} -
- {% csrf_token %} - - - - - -
-
-{% endblock content %} -{% block javascripts %} - - - - - -{% endblock javascripts %}