Cosmos rewiring

.coveragerc

@@ -0,0 +1,6 @@
+[run]
+source = .
+include = */*.py
+
+[report]
+show_missing = True

.env_sample

@@ -6,13 +6,33 @@ DJANGO_ACCOUNT_ALLOW_REGISTRATION=False
 DJANGO_AWS_ACCESS_KEY_ID=""
 DJANGO_AWS_SECRET_ACCESS_KEY=""
 DJANGO_AWS_STORAGE_BUCKET_NAME=""
-GITHUB_ACCESS_TOKEN=""
-GITHUB_BRANCH_FOR_WEBAPP=""
 IPYTHONDIR=""
 REDIS_URL=""
-SINEQUA_CONFIGS_GITHUB_REPO=""
-SINEQUA_CONFIGS_REPO_DEV_BRANCH=""
-SINEQUA_CONFIGS_REPO_MASTER_BRANCH=""
-SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH=""
 SLACK_WEBHOOK_URL=""
 USE_DOCKER=no
+
+# SDE curation pipeline (all optional — defaults hold when unset)
+AWS_REGION="us-east-1"
+SDE_S3_BUCKET=""
+CRAWLER_INSTANCE_ID=""
+CRAWLER_INBOX_PATH="/opt/sde-crawler/jobs/incoming"
+SCRAPE_POLL_ENABLED=False
+SCRAPE_STALL_TIMEOUT_HOURS=24
+INFERENCE_ENABLED=False
+# pipeline-scoped credentials for local dev ONLY; leave blank in AWS (instance role takes over)
+SDE_AWS_ACCESS_KEY_ID=""
+SDE_AWS_SECRET_ACCESS_KEY=""
+# only with temporary creds: aws configure export-credentials --profile sde-dev --format env
+SDE_AWS_SESSION_TOKEN=""
+
+# P7 indexing hand-off — dev-only for now (sde-web-copy); blank = dispatch disabled
+SDE_INDEX_BUCKET=""
+INDEXING_ECS_CLUSTER=""
+INDEXING_TASK_FAMILY=""
+INDEXING_CONTAINER_NAME="WEB_COSMOSContainer"
+INDEXING_DISPATCH_ROLE_ARN=""
+INDEXING_SUBNETS=""
+INDEXING_SECURITY_GROUPS=""
+INDEXING_ASSIGN_PUBLIC_IP=True
+INDEX_POLL_ENABLED=False
+INDEX_STALL_TIMEOUT_HOURS=6

.envs/.local/.django

@@ -22,28 +22,37 @@ DJANGO_AWS_ACCESS_KEY_ID=''
 DJANGO_AWS_SECRET_ACCESS_KEY=''
 DJANGO_AWS_STORAGE_BUCKET_NAME=''
 
-# GitHub (please create a new file called .env and put these in there)
-# ------------------------------------------------------------------------------
-GITHUB_ACCESS_TOKEN=
-SINEQUA_CONFIGS_GITHUB_REPO='NASA-IMPACT/sde-backend'
-SINEQUA_CONFIGS_REPO_MASTER_BRANCH='master'
-SINEQUA_CONFIGS_REPO_DEV_BRANCH='dev'
-SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH='dummy_branch'
-
 # Slack Webhook
 # ------------------------------------------------------------------------------
 SLACK_WEBHOOK_URL=''
 
-#Server Credentials
-#--------------------------------------------------------------------------------
-LRM_DEV_USER=''
-LRM_DEV_PASSWORD=''
-XLI_USER=''
-XLI_PASSWORD=''
-LRM_QA_USER=''
-LRM_QA_PASSWORD=''
-
-#Server Tokens
-#--------------------------------------------------------------------------------
-LRM_DEV_TOKEN=''
-XLI_TOKEN=''
+# SDE curation pipeline
+# All optional — defaults hold when unset. SDE_AWS_* are pipeline-scoped credentials
+# for local dev ONLY (distinct from DJANGO_AWS_* static-assets credentials above);
+# leave blank in AWS, where the instance role takes over.
+# ------------------------------------------------------------------------------
+AWS_REGION='us-east-1'
+SDE_S3_BUCKET=''
+CRAWLER_INSTANCE_ID=''
+CRAWLER_INBOX_PATH='/opt/sde-crawler/jobs/incoming'
+SCRAPE_POLL_ENABLED=False
+SCRAPE_STALL_TIMEOUT_HOURS=24
+INFERENCE_ENABLED=False
+# Real keys and the dev resource ids go in the gitignored .envs/.local/.sde-aws (see
+# .sde-aws.sample); local.yml loads it after this file, so its values win. This file is tracked.
+SDE_AWS_ACCESS_KEY_ID=''
+SDE_AWS_SECRET_ACCESS_KEY=''
+SDE_AWS_SESSION_TOKEN=''
+
+# P7 indexing hand-off — dev-only for now (sde-web-subset); blank = dispatch disabled
+SDE_INDEX_BUCKET=''
+INDEXING_ECS_CLUSTER=''
+INDEXING_TASK_FAMILY=''
+INDEXING_CONTAINER_NAME='WEB_COSMOSContainer'
+# Blank on a laptop: the dispatch role only trusts the instance role, so a local SSO
+# session calls ecs:RunTask with SDE_AWS_* directly. Set in AWS.
+INDEXING_DISPATCH_ROLE_ARN=''
+INDEXING_SUBNETS=''
+INDEXING_SECURITY_GROUPS=''
+INDEX_POLL_ENABLED=False
+INDEX_STALL_TIMEOUT_HOURS=6

.envs/.local/.sde-aws.sample

@@ -0,0 +1,22 @@
+# Copy to .envs/.local/.sde-aws (gitignored). local.yml loads it after .django, so
+# anything set here overrides the tracked blanks/defaults.
+
+# Temporary SSO credentials. Refresh with:
+#   aws sso login --profile sde-dev && aws configure export-credentials --profile sde-dev --format env
+# then recreate the containers (docker compose -f local.yml up -d --force-recreate django celeryworker celerybeat).
+SDE_AWS_ACCESS_KEY_ID=
+SDE_AWS_SECRET_ACCESS_KEY=
+SDE_AWS_SESSION_TOKEN=
+
+# Dev pipeline resources — ask a teammate for the current ids; they are not committed.
+SDE_S3_BUCKET=
+CRAWLER_INSTANCE_ID=
+SDE_INDEX_BUCKET=
+INDEXING_ECS_CLUSTER=
+INDEXING_TASK_FAMILY=
+INDEXING_SUBNETS=
+INDEXING_SECURITY_GROUPS=
+
+# Optional: run the S3 pollers on this machine (tracked default is off).
+# SCRAPE_POLL_ENABLED=True
+# INDEX_POLL_ENABLED=True

.github/ISSUE_TEMPLATE/bug_report.md

@@ -11,7 +11,7 @@ assignees: ''
 A clear and concise description of what the bug is.
 
 ## Steps To Reproduce
-Steps to reproduce the behavior on https://sde-indexing-helper.nasa-impact.net/:
+Steps to reproduce the behavior on the COSMOS web application:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'

.github/workflows/run_full_test_suite.yml

@@ -22,23 +22,23 @@ jobs:
       - name: Check out merged code
         uses: actions/checkout@v2
 
-      - name: Set up Docker Compose
-        run: |
-          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-          sudo chmod +x /usr/local/bin/docker-compose
+      # Compose v2 ships with the runner's Docker. local.yml uses the long-form
+      # optional env_file (Compose >= 2.24), which the EOL v1 binary rejects.
+      - name: Show Docker Compose version
+        run: docker compose version
 
       - name: Build the Docker environment
-        run: docker-compose -f local.yml build
+        run: docker compose -f local.yml build
 
       - name: Run test suite
         env:
           DJANGO_ENV: test
-        run: docker-compose -f local.yml run --rm django bash ./init.sh
+        run: docker compose -f local.yml run --rm django bash ./init.sh
 
       - name: Generate Coverage Report
         env:
           DJANGO_ENV: test
-        run: docker-compose -f local.yml run --rm django bash -c "coverage report"
+        run: docker compose -f local.yml run --rm django bash -c "coverage report"
 
       - name: Cleanup
-        run: docker-compose -f local.yml down --volumes
+        run: docker compose -f local.yml down --volumes

.gitignore

@@ -282,6 +282,7 @@ sde_indexing_helper/media/
 .env
 .envs/*
 !.envs/.local/
+.envs/.local/.sde-aws
 
 **/.ipynb_checkpoints/
 **/*.xlsx

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 exclude: "^docs/|/migrations/"
-default_stages: [commit]
+default_stages: [pre-commit]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -12,7 +12,7 @@ repos:
       - id: debug-statements
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
+    rev: v3.21.2
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
@@ -61,7 +61,7 @@ repos:
           - types-requests
 
   - repo: https://github.com/PyCQA/bandit
-    rev: "1.7.0"
+    rev: "1.8.6"
     hooks:
       - id: bandit
         args: ["-r", "--configfile=bandit-config.yml"]

CHANGELOG.md

@@ -13,6 +13,17 @@ For each PR made, an entry should be added to this changelog. It should contain
 
 ## Changelog
 ### 3.1.??
+- cosmos-rewiring review fixes
+  - Description: Pre-merge fixes from the branch review. Status-triggered tasks are enqueued on commit; Slack messages are posted for statuses set via queryset `.update()` and for the `PRODUCTION_INDEXING -> PROD_*` transitions; re-scrape failures no longer overwrite the live workflow status; `send_job_to_crawler` refuses to run on a host without `CRAWLER_INSTANCE_ID`; duplicate URLs in crawl output are dropped instead of aborting the ingest; the prod status mirror map is explicit; `print()` replaced by `logging` in `tasks.py`.
+  - Changes:
+    - `sde_collections/models/collection.py`: `_enqueue_on_commit` wraps the four `.delay()` calls in `handle_workflow_status_change`
+    - `sde_collections/tasks.py`: `_mark_scrape_failed`, `_dedupe_by_url`, `PROD_STATUS_FOR_QC_STATUS`, logging
+    - `sde_collections/utils/slack_utils.py`: `notify_status_change`, two new `PRODUCTION_INDEXING -> PROD_*` messages
+    - `sde_collections/scraping/ssm_dispatch.py`: settings guard
+    - `templates/sde_collections/collection_detail.html`: removed the "View on prod" buttons (properties deleted with Sinequa)
+    - New tests: `test_signals.py`, `test_management_commands.py`; additions to the scrape/ingest/indexing/trigger suites
+    - `gitleaks-config.toml` added so the pre-commit gitleaks hook runs (it referenced a missing file)
+    - Deployment: none beyond the branch's existing migration/env steps
 - 1232-process-the-full-text-dump
   - Description: A script was added `/scripts/sde_dump_processing/clean_text_dump.py` which cleans dumps from sinequa. The sinequa dump does not respect normal csv new line formatting, so that a dump of 1.8 million records becomes a csv of 900 million lines. This script can detect the headers and process the dump with the three possible sources TDAMM, SDE, and scripts, in order to create a final, clean csv. It has a simple CLI which allows setting the input and output, the verbosity of the logs, etc. Because the input files can be very large, the script streams them instead of holding them in memory.
   - Changes:

CODE_STANDARDS.md

@@ -47,10 +47,9 @@ The following pre-commit hooks are configured:
 - black: Formats Python code to ensure consistent styling.
 - isort: Sorts imports alphabetically and automatically separated into sections.
 - flake8: Lints code to catch styling errors and potential bugs.
-- mypy: Checks type annotations to catch potential bugs.
+- mypy: Configured to check type annotations, but currently a no-op — the hook sets `exclude: "."`, so no files are checked.
 - bandit: Scans code for common security issues.
-- gitleaks: Prevents secrets from being committed to the repository.
-- hadolint: Lints Dockerfiles to ensure best practices and common conventions are followed.
+- gitleaks: Intended to prevent secrets from being committed to the repository. Known gap: the hook passes `--config=gitleaks-config.toml`, and that file does not exist in the repository, so the hook fails instead of scanning.
 
 ## Continuous Integration (CI)
 When a commit is pushed to a branch that is part of a Pull Request, our Continuous Integration (CI) pipeline automatically runs specified tools to check code quality, style, security and other standards. If these checks fail, the PR cannot be merged until all issues are resolved.