diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index c12eaff1..c0d04aaa 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -184,6 +184,8 @@ jobs: run: pip install -r requirements.txt coverage - name: Install Playwright run: playwright install --with-deps chromium + - name: Collect static files + run: python manage.py collectstatic --noinput --verbosity 0 - name: Run E2E tests run: | export SECRET_KEY=$(python -c 'import secrets; print(secrets.token_hex(50))') diff --git a/.github/workflows/opencode.yml b/.github/workflows/opencode.yml new file mode 100644 index 00000000..eec65625 --- /dev/null +++ b/.github/workflows/opencode.yml @@ -0,0 +1,124 @@ +name: opencode + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + +jobs: + opencode: + if: | + github.event.comment.user.type != 'Bot' && + (contains(github.event.comment.body, '/oc') || + contains(github.event.comment.body, '/opencode')) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + id-token: write + actions: write + contents: write + steps: + - name: job-start + run: echo "job-start:$(date -u +%Y-%m-%dT%H:%M:%SZ) sha:${{ github.sha }}" + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Configure git identity + run: | + git config --global user.name "opencode-agent[bot]" + git config --global user.email "opencode-agent[bot]@users.noreply.github.com" + + # Install the same commit-msg hook humans get via scripts/commit-msg.sh + # (see .pre-commit-config.yaml), so opencode's own `git commit` gets + # rejected with the actual regex+reason inline and can retry, instead + # of only finding out after pushing. Only the commit-msg hook type -- + # not the full pre-commit stage, which runs unrelated file-content + # checks that could block a commit for issues in files opencode + # didn't touch. + - name: Install commit-msg hook + run: | + pip install --quiet pre-commit + pre-commit install --hook-type commit-msg + + - name: Run OpenCode + uses: anomalyco/opencode/github@31406ccc51b4bd2a4e1e086b2bcaa5f7f804f26d # v1.18.18 + env: + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + with: + # Model routing by comment tag: default is the free Zen model; + # add [security] or [feature] anywhere in the comment to route to + # NVIDIA NIM's nemotron-3-ultra (1M context) instead, e.g. + # "/oc [security] fix this CVE". build.nvidia.com labels this + # endpoint "Free Endpoint" in its own product metadata — do not + # trust models.dev's cost field for NVIDIA NIM models without + # cross-checking build.nvidia.com directly, it has been wrong here. + # The bracket tag is required for THIS routing expression to match + # ("/oc-security ..." does not contain "[security]" so it silently + # falls through to the default model below) — confirmed live on a + # sibling repo (fawkes, issue #1587): "/oc-security ..." does still + # trigger the action, just without escalating the model. + # + # Model string is doubled ("nvidia/nvidia/...") deliberately, not a + # typo: NVIDIA-published models on NVIDIA's own NIM catalog carry + # "nvidia/" as part of the model's own org/name identifier (an + # org/model convention, same as e.g. "microsoft/phi-4-..." models + # hosted on the same catalog) -- confirmed against models.dev's raw + # JSON directly (data['nvidia']['models'] keys). opencode splits on + # the FIRST "/" only for provider vs. model, so referencing this + # specific model needs provider "nvidia" + catalog key + # "nvidia/nemotron-3-ultra-550b-a55b" == "nvidia/nvidia/...". + model: ${{ (contains(github.event.comment.body, '[security]') || contains(github.event.comment.body, '[feature]')) && 'nvidia/nvidia/nemotron-3-ultra-550b-a55b' || 'opencode/deepseek-v4-flash-free' }} + + # opencode does not reliably follow Conventional Commits format even + # with documentation in place (confirmed on a sibling repo, fawkes, + # PRs #1619 and #1625). Documentation alone isn't reliable enough; + # enforce mechanically as a backstop in case the commit-msg hook above + # doesn't catch it (e.g. opencode retries and still gets it wrong). + - name: Normalize commit message if non-compliant + if: always() + run: | + set -euo pipefail + if ! git rev-parse --verify HEAD >/dev/null 2>&1; then + echo "No commits to check." + exit 0 + fi + if [ "$(git rev-parse HEAD)" = "${{ github.sha }}" ]; then + echo "No new commits; nothing to normalize." + exit 0 + fi + subject="$(git log -1 --format=%s)" + if echo "$subject" | grep -qE '^(feat|fix|docs|style|refactor|test|chore|ci|perf|build|revert)(\(.+\))?: .{1,72}$'; then + echo "Commit message already compliant: $subject" + exit 0 + fi + echo "Non-compliant commit subject, rewriting: $subject" + desc="$subject" + if [ "${#desc}" -gt 69 ]; then + desc="${desc:0:69}..." + fi + body="$(git log -1 --format=%b)" + { + echo "fix: ${desc}" + echo + echo "Original subject: ${subject}" + if [ -n "$body" ]; then + echo + echo "$body" + fi + } > /tmp/normalized_commit_msg.txt + git commit --amend -F /tmp/normalized_commit_msg.txt + git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" + git push --force-with-lease + env: + GH_TOKEN: ${{ github.token }} + + - name: job-finish + if: always() + run: echo "job-finish:$(date -u +%Y-%m-%dT%H:%M:%SZ)" diff --git a/.opencode/skills/commit-message-format/SKILL.md b/.opencode/skills/commit-message-format/SKILL.md new file mode 100644 index 00000000..879036c5 --- /dev/null +++ b/.opencode/skills/commit-message-format/SKILL.md @@ -0,0 +1,48 @@ +--- +name: commit-message-format +description: Use before running `git commit` on this repo — the subject line format is enforced by CI and a bad one blocks the whole PR from merging. +--- + +# Commit Message Format + +This repo's `ci-commit-lint.yml` check runs this exact regex against every +commit in a PR (not just the PR title, not just the latest commit — every +one): + +``` +^(feat|fix|docs|style|refactor|test|chore|ci|perf|build|revert)(\(.+\))?: .{1,72}$ +``` + +One commit that doesn't match fails the whole PR's Commit Lint check, even +if every other commit and the actual code change is fine. + +## Before running `git commit` + +1. Pick a `type`: `feat fix docs style refactor test chore ci perf build revert`. +2. Optionally add `(scope)` — a short parenthesized area, e.g. `(security)`, `(ci)`. +3. Write the description in **1–72 characters**. Count it — don't guess. A + precise-but-long subject ("fix(ci): correct the model provider id, wire + up multiple new provider keys, and add debugging skills") still fails. + Move detail into the commit body below the subject line instead. + +## Quick self-check + +```bash +python3 -c " +import re, sys +regex = re.compile(r'^(feat|fix|docs|style|refactor|test|chore|ci|perf|build|revert)(\(.+\))?: .{1,72}\$') +subject = sys.argv[1] +print(len(subject), bool(regex.match(subject))) +" "type(scope): your subject here" +``` + +If it prints `False`, shorten the description (or fix the type) before +committing — don't push and let CI catch it after the fact. + +## If a commit was already pushed with a bad subject + +The message must be reworded and the branch force-pushed — there's no way to +fix a commit's message without rewriting its hash. Reuse the exact tree +(`git commit-tree -p -F `) so the +code diff is provably unchanged, verify the new subject against the regex +above, then force-push with `--force-with-lease`. diff --git a/.opencode/skills/root-cause-debugging/SKILL.md b/.opencode/skills/root-cause-debugging/SKILL.md new file mode 100644 index 00000000..3c1a2248 --- /dev/null +++ b/.opencode/skills/root-cause-debugging/SKILL.md @@ -0,0 +1,37 @@ +--- +name: root-cause-debugging +description: Use before proposing any fix for a failing test, CI job, security scan finding, or bug report. Front-load root-cause investigation over guessing. +--- + +# Root-Cause Debugging + +**No fix without root-cause investigation first.** A fix that only makes the +symptom disappear (widening a try/except, bumping a timeout, pinning around +an error instead of understanding it) is not a fix — it is deferred rework, +and on this repo it gets caught in review. + +## Before touching code + +1. Read the full error/log output, not just the last line — stack traces and + CI job logs usually name the exact failing line and cause. +2. Reproduce it. If you can't reproduce it, gather more evidence before + guessing (add logging, run the failing step in isolation) — don't ship a + fix for a failure you can't trigger. +3. Check what changed: `git log`/`git blame`/`git diff` on the affected file + and its recent commits. +4. For multi-component failures (CI → build → deploy, API → service → DB), + trace which layer actually breaks before fixing any of them — add a log + line at each boundary if needed. +5. Grep every caller of the function/config you're about to touch. The + root-cause fix is usually the smaller diff: one guard in the shared + function beats patching every call site that happens to hit it. + +## Fixing + +- State the root cause in one sentence before writing the fix. +- Make the smallest change that addresses that cause — no bundled + refactors, no "while I'm here" cleanup. +- If a fix doesn't resolve the issue, don't stack a second fix on top — + re-open the investigation. Three failed fixes in a row on the same issue + means the architecture is wrong, not that you need a fourth attempt — + stop and say so in the PR/issue instead of continuing to guess. diff --git a/.opencode/skills/security-fix-guardrails/SKILL.md b/.opencode/skills/security-fix-guardrails/SKILL.md new file mode 100644 index 00000000..ae231740 --- /dev/null +++ b/.opencode/skills/security-fix-guardrails/SKILL.md @@ -0,0 +1,48 @@ +--- +name: security-fix-guardrails +description: Use before implementing any issue labeled type-security or comp-security, or any fix touching auth, RBAC, secrets, IAM, security groups, network policy, or crypto. +--- + +# Security Fix Guardrails + +Most `type-security` issues in this repo are mechanical and safe to fix +directly: add a missing `securityContext` block, bump a dependency to a +patched version, add encryption/`readOnlyRootFilesystem` to a Terraform or +K8s resource. Fix those the same way as any other issue — root cause, small +diff, verify. + +A smaller set need a human in the loop before or instead of an auto-fix. +Route to that path whenever the issue or the fix touches: + +- Authentication, authorization, or RBAC logic +- Secrets, credentials, tokens, or any `ConfigMap`/`Secret` handling +- IAM policies, security groups, network ingress/egress rules +- Cryptographic code (hashing, signing, encryption implementations) +- Anything already flagged in the issue as "inspect before fixing" or similar + +For these: still do the investigation and propose the fix, but say so +explicitly in the PR description or comment — call out that it's a +security-sensitive change needing manual review rather than opening it as if +it were routine. Never merge a security-sensitive change yourself even if +`/oc` has write access to open PRs. + +## Repo-specific rules that already cover this + +- `AGENTS.md` §9: any `infra/` change requires a second human reviewer — + applies to every Terraform security fix (SG rules, IAM, encryption), not + just infra changes in general. +- `AGENTS.md` §12: "Auth, RBAC, secrets, or any infra-touching change" is + security-sensitive by default. +- Never widen a `try`/`except` or add a broad exception handler to make a + security scanner finding disappear — that hides the finding, it doesn't + fix it. If a scanner flag looks like a false positive, say so and explain + why instead of suppressing it. + +## Verifying a security fix specifically + +Beyond the normal test/lint run (see `verify-before-done`): +- For a CVE/dependency bump: confirm the new version range actually excludes + the vulnerable version (check the advisory, don't just bump and assume). +- For a K8s/Terraform hardening change: run `helm template`/`terraform plan` + and confirm the rendered output actually has the intended field — a + misplaced YAML key silently no-ops the fix. diff --git a/.opencode/skills/tdd-workflow/SKILL.md b/.opencode/skills/tdd-workflow/SKILL.md new file mode 100644 index 00000000..affd6d7d --- /dev/null +++ b/.opencode/skills/tdd-workflow/SKILL.md @@ -0,0 +1,48 @@ +--- +name: tdd-workflow +description: Use before implementing any bug fix, new function, or config change — write and run a failing test first, confirm it fails for the right reason, then implement to pass. +--- + +# Test-Driven: Red, Then Green + +**Never write implementation before a test that proves it's needed.** This +applies to bug fixes, new functions, and infra/config changes alike — not +just application code. + +## The loop + +1. **Write the test first.** For a bug: reproduce it. For new behavior: assert + the contract you're about to build. For infra/config: the check that + proves the config has the intended effect (see commands below). +2. **Run it. Confirm it fails — and read *why*.** A test that fails for the + wrong reason (typo, missing import, wrong file path) proves nothing. Fix + the test setup until the failure is the one you actually expect, before + touching implementation. +3. **Write the minimum code to pass.** No extra scope, no "while I'm here" + changes — see the surgical-changes rule in `AGENTS.md`. +4. **Run it again. Confirm green.** Then run the broader suite for that area, + not just the one test — see `verify-before-done`. + +Never write the test after the code to match what the code already does — +that only proves the code does what it does, not that it's correct. + +## Commands by area (this repo) + +| Area | Command | +|---|---| +| Python | `pytest ` (or `pytest --cov=. --cov-report=term-missing` for coverage) | +| Shell scripts | `make test-bats` (single file: `bats tests/.bats`) | +| Terraform modules | `make terraform-test` (unit-level, no resources deployed) | +| BDD / acceptance | `make test-bdd COMPONENT=` | +| Everything | `make test-unit` then `make test-all` before opening a PR | + +For a Kubernetes/Helm config change with no test framework, the equivalent +red/green check is `helm template` / `kubeconform -strict` before vs. after — +render it first, confirm the field you're adding is absent (red), add it, +confirm it's present in the rendered output (green). + +## When there's no existing test framework for the file + +Still do it: a small standalone script (`scripts/`) or a one-off assertion +block is enough — see `AGENTS.md` §7's per-language required checks for what +already exists to build on before inventing a new pattern. diff --git a/.opencode/skills/verify-before-done/SKILL.md b/.opencode/skills/verify-before-done/SKILL.md new file mode 100644 index 00000000..fdd2b352 --- /dev/null +++ b/.opencode/skills/verify-before-done/SKILL.md @@ -0,0 +1,34 @@ +--- +name: verify-before-done +description: Use before claiming a fix works, tests pass, or a PR is ready — before committing, pushing, or replying "fixed" on an issue/PR comment. +--- + +# Verify Before Claiming Done + +**Evidence before claims.** Don't tell a human (or write in a commit/PR/issue +comment) that something is fixed, passing, or ready unless you just ran the +command that proves it, in this session, and read its output. + +## Gate before any completion claim + +1. Identify the exact command that proves the claim (test suite, linter, + `terraform validate`, `helm lint`, the specific failing CI job re-run). +2. Run it fresh — not "ran it earlier," not "should still pass." +3. Read the full output and exit code, not just the last line. +4. Only then state the claim, citing what you ran and its result. + +| Claim | Needs | Not enough | +|---|---|---| +| Tests pass | Fresh test run, 0 failures | "Should pass now" | +| Bug fixed | Test for the original symptom, now green | Code changed, assumed fixed | +| Lint/build clean | Fresh linter/build output, exit 0 | Linter passed (≠ compiler/build) | +| PR ready | Checklist against the issue's acceptance criteria | Tests passing alone | + +## On this repo specifically + +- `make lint` output goes in the PR body per `AGENTS.md` §8 — run it, don't + paraphrase it. +- Never delete or weaken a test to make CI green; if a test seems wrong, say + so and ask, don't route around it. +- If you can't run something in this environment (e.g. no live cluster), say + exactly that instead of asserting success. diff --git a/docs/assessments/PRODUCT_AUDIT_2026-08-19.md b/docs/assessments/PRODUCT_AUDIT_2026-08-19.md new file mode 100644 index 00000000..61aad80b --- /dev/null +++ b/docs/assessments/PRODUCT_AUDIT_2026-08-19.md @@ -0,0 +1,138 @@ +# Product Audit — prei Residential RE Investing Workflows + +**Date**: 2026-08-19 +**Auditor**: AI-assisted review against PRODUCT_STRATEGY.md maturity matrix + +--- + +## Executive Summary + +prei is a **passive residential real estate investment analytics** platform for buy-and-hold investors. The core workflow spans Growth Areas → Discovery → Screening → Underwriting → Offer → Pipeline CRM → Portfolio → Leasing. + +Current state: Most components are **ALPHA to BETA** with significant gaps in acceptance testing, UX polish, and end-to-end workflow verification. + +--- + +## Maturity Assessment (vs. PRODUCT_STRATEGY.md July 2026) + +| Component | Current | Target | Key Gaps | +|-----------|---------|--------|----------| +| **Growth Areas** | BETA | GA | Acceptance tests, seed data, data health dashboard | +| **Property Discovery** | ALPHA→BETA | BETA | Scraper reliability, saved searches, source health status | +| **Screening** | ALPHA (thin) | BETA | Batch filter/sort UX, criteria settings page, acceptance tests | +| **Underwriting** | BETA | GA | Side-by-side deal comparison, market cap rate in reports | +| **Offer Management** | ALPHA (thin) | BETA | MAO visualization, competition multiplier UX | +| **Pipeline CRM** | ALPHA (thin) | BETA | **Drag-and-drop kanban** (biggest UX gap) | +| **Portfolio Tracking** | BETA | GA | Time-series charts, equity dashboard | +| **BRRRR** | ALPHA (thin) | BETA | Visual projection timeline, equity recycle log | +| **Leasing Pipeline** | THIN SHELL | BETA | Backend wiring for kanban (drag-drop + API) | + +--- + +## Top 7 Improvement Opportunities (Priority Order) + +### **P0 — Pipeline CRM Kanban (Biggest UX Gap)** +- **Current**: List view only, no drag-and-drop, no stage transition API +- **Template exists**: `templates/pipeline/kanban.html` has full drag-drop markup + JS but **no backend endpoints** +- **Need**: `POST /pipeline//advance/`, `POST /pipeline//kill/`, `POST /pipeline//hold/` endpoints + WebSocket/polling for real-time multi-user + +### **P0 — Property Discovery Data Health** +- **Scrapers unreliable**: 11 TX counties, sheriff sales, ATTOM — no monitoring/alerting +- **No source health dashboard**: User can't see if HUD/USDA/VRM/County data is fresh +- **Need**: `DataSourceHealth` model + scheduled ingestion status + UI status page (partially in `system_status` view) + +### **P1 — Screening Batch UX + Criteria Settings** +- **Screener page** (`/pipeline/screener/`) exists but **no filter/sort controls** on results table +- **Criteria page** (`/pipeline/screening-settings/`) — form exists but no live preview of kill impact +- **Need**: HTMX-powered filter bar, "simulate screening" button, save/version criteria + +### **P1 — Leasing Pipeline Backend Wiring** +- **Kanban template** exists (`templates/leasing/kanban.html`) with columns: `LISTED → SCREENING → APPROVED → LEASED` +- **Zero backend**: No stage transition API, no tenant screening workflow, no lease document storage +- **Need**: Leasing stage model + API + tenant application form + e-sign integration + +### **P2 — Deal Comparison UI (Underwriting)** +- **Math is solid** (58 parametrized tests verified), but **no side-by-side comparison** +- **Need**: Multi-property comparison view with aligned KPI columns, variance highlighting, export + +### **P2 — Growth Areas Acceptance Tests + Seeded Data** +- **Works but empty on deploy**: Requires `populate_growth_areas` CLI command +- **No automated acceptance tests** against deployed artifact +- **Need**: Seed script for TX/FL/GA default markets, Playwright tests for Growth Explorer flow + +### **P3 — BRRRR Visual Timeline + Equity Log** +- **Calculator works** (client + server), but **no visual projection** +- **Need**: Year-by-year equity recycle chart, cash-out refi waterfall, sensitivity sliders + +--- + +## Code Quality Observations + +| Area | Issue | Severity | +|------|-------|----------| +| **Duplication** | `create_from_vrm/foreclosure/hud/usda/county` in `pipeline.py` — 5 nearly identical functions | Medium | +| **Hardcoded thresholds** | Screening weights (20/15/10/5 pts) in `screening.py` — not user-configurable | Low | +| **Missing indexes** | `PipelineProperty` queries filter by `user+stage+status` — no composite index | Medium | +| **N+1 in portfolio** | `compute_portfolio_performance` loops properties → queries `MonthlyActuals` each | High | +| **Test gaps** | **No Playwright E2E tests for any full workflow (discovery→screen→underwrite→offer)** | **Critical** | + +--- + +## Recommended Implementation Sequence + +### Phase 1: TDD Foundation (Week 1) +1. **Playwright E2E test suite** for full workflow: Growth Explorer → Discovery → Screening → Underwriting → Offer → Pipeline +2. CI integration with headless browser + +### Phase 2: Pipeline Kanban Backend (Week 2) +1. Stage transition API endpoints (`advance`, `kill`, `hold`, `reactivate`) +2. WebSocket/polling for real-time updates +3. Drag-drop frontend already exists — wire it up + +### Phase 3: Data Health & Screening UX (Week 3) +1. DataSourceHealth model + scheduled ingestion monitoring +2. Screening filter/sort + criteria simulation + +### Phase 4: Leasing & Comparison (Week 4+) +1. Leasing pipeline backend +2. Deal comparison UI + +--- + +## Files Referenced in Audit + +### Core Services +- `core/services/market_scoring.py` — GACS composite scoring +- `core/management/commands/populate_growth_areas.py` — Growth area population +- `core/models/growth.py` — GrowthArea, MarketSnapshot models +- `core/services/sources/registry.py` — Discovery source registry (6 sources) +- `core/services/screening.py` — 9-criteria screening (4 hard kill, 5 soft) +- `core/services/underwriting.py` — NOI, Cap Rate, CoC, MAO solver +- `core/services/scoring.py` — v2 investor-grade underwriting score (6 signals) +- `core/services/offer.py` — Offer price optimization (3 strategies) +- `core/services/pipeline.py` — Pipeline lifecycle (11 stages, 5 source creators) +- `core/services/brrrr.py` — BRRRR analysis (pure math + Django orchestrator) +- `core/services/portfolio.py` — Portfolio aggregates, variance analysis + +### Finance Math (verified 58 parametrized tests) +- `investor_app/finance/utils.py` — Core KPIs (NOI, Cap Rate, CoC, DSCR, IRR) +- `investor_app/finance/mortgage.py` — Monthly mortgage, carrying costs +- `investor_app/finance/taxes.py` — Depreciation, after-tax cashflow +- `investor_app/finance/strategies.py` — BRRRR, flip, buy-and-hold math + +### Templates (key workflow pages) +- `templates/growth_explorer.html` — State picker, results table +- `templates/property_discovery.html` — Source selection, discovery modal +- `templates/pipeline/screener.html` — Screening results table +- `templates/pipeline/kanban.html` — **Drag-drop kanban (frontend only)** +- `templates/brrrr_calculator.html` — Client-side BRRRR calculator +- `templates/leasing/kanban.html` — Leasing kanban (frontend only) + +--- + +## Next Actions + +1. **Save this audit** → `docs/assessments/PRODUCT_AUDIT_2026-08-19.md` ✅ +2. **Create implementation plan** with `writing-plans` skill +3. **Start TDD**: Write Playwright E2E tests for discovery→screen→underwrite→offer +4. **Then**: Implement Pipeline Kanban drag-drop backend diff --git a/docs/superpowers/plans/2026-08-20-playwright-e2e-foundation.md b/docs/superpowers/plans/2026-08-20-playwright-e2e-foundation.md new file mode 100644 index 00000000..74b719f4 --- /dev/null +++ b/docs/superpowers/plans/2026-08-20-playwright-e2e-foundation.md @@ -0,0 +1,536 @@ +# Playwright E2E Foundation — Full Workflow & Kanban Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a real-browser Playwright E2E test suite covering the full prei workflow (Growth Explorer → Discovery → Screening → Underwriting → Offer → Pipeline) plus the Pipeline Kanban drag-and-drop interaction, and wire it into CI, so the audit's Phase 1 "Critical: No Playwright E2E tests" gap is closed. + +**Architecture:** A new `tests/e2e/` package launches headless Chromium (Playwright sync API) against pytest-django's `live_server`. All source data (VRM/HUD/USDA/County) is seeded via Django fixtures so discovery never touches the live network. Two independent browser test modules cover (1) the end-to-end property journey and (2) kanban drag-and-drop stage advancement. The existing `tests-e2e` CI job (which already installs Chromium and runs `-m e2e`) picks up the new tests; it gains a `collectstatic` step for realistic rendering. + +**Tech Stack:** Playwright 1.62 (sync API), pytest 9.1 + pytest-django 4.12 (`live_server`), Django 6.0, GitHub Actions (`ci-quality.yml`). + +## Global Constraints + +- No external network calls in browser tests — every source model that discovery reads is seeded first (the discovery view only scrapes when a source table is empty). +- Currency and rates are `Decimal` everywhere in fixtures and assertions. +- All new test files live under `tests/e2e/`; every test module sets `pytestmark = pytest.mark.django_db(transaction=True)` (required for `live_server` with file-based SQLite — mirrors `tests/acceptance/conftest.py`). +- Test modules are named with the `_e2e.py` suffix so the root `conftest.py` auto-assigns the `e2e` marker (CI runs `-m e2e`). +- Do not use Bootstrap classes, inline `style=` layout attributes, or `!important` (repo rule) — none needed here since tests only assert on DOM text/attributes. +- Commit messages follow Conventional Commits (`test:`, `ci:`, `docs:`) — enforced by CI. + +--- + +### Task 1: Playwright Browser Harness + Smoke Test + +**Files:** +- Create: `tests/e2e/__init__.py` +- Create: `tests/e2e/conftest.py` +- Create: `tests/e2e/test_harness_e2e.py` +- Create: `docs/superpowers/plans/README.md` (one-line index of this plan — optional, skip if the dir already has an index) + +**Interfaces:** +- Consumes: root `conftest.py` (auto `e2e` marker for `_e2e.py` files), `investor_app.settings_test` (file-based SQLite enabled for `live_server`), `playwright==1.62.0` from `requirements.txt`. +- Produces: + - `tests/e2e/conftest.py` fixtures consumed by Tasks 2–3: + - `browser` → `playwright.sync_api.Browser` (session-scoped, headless Chromium) + - `page` → `playwright.sync_api.Page` (function-scoped, bound to `live_server`) + - `e2e_user` → `django.contrib.auth.User` (username `e2e_user`, password `e2e-password-123`) + - `e2e_login` → logs the user in through the real `/accounts/login/` form, returns the `User` + - `growth_area` → `core.models.GrowthArea` (Austin, TX, GACS 75.50) + - `discovery_sources` → list of `[VrmProperty, HudProperty, UsdaProperty, CountyForeclosureNotice]` all in Austin, TX + - `tests/e2e/test_harness_e2e.py` `TestHarness` with `test_page_loads_health_check` and `test_login_redirects_to_dashboard`. + +- [ ] **Step 1: Create the package files** + +`tests/e2e/__init__.py`: +```python +"""Browser-based E2E tests (Playwright).""" +``` + +- [ ] **Step 2: Write the failing smoke test** + +`tests/e2e/test_harness_e2e.py`: +```python +"""Smoke tests proving the Playwright harness binds to live_server and can log in.""" + +import pytest + +pytestmark = pytest.mark.django_db(transaction=True) + + +class TestHarness: + def test_page_loads_health_check(self, page) -> None: + page.goto("/health/") + assert page.locator("body").inner_text() != "" + + def test_health_returns_ok(self, page) -> None: + page.goto("/health/") + assert "ok" in page.locator("body").inner_text() + + def test_login_redirects_to_dashboard(self, page, e2e_login) -> None: + page.goto("/dashboard/") + assert page.locator("text=Dashboard").count() >= 1 or page.url.endswith("/dashboard/") +``` + +- [ ] **Step 3: Run the harness smoke test to verify failure** + +Run from repo root (assuming `.venv` is active; also requires `playwright install chromium` to have been run once locally): +``` +.venv/bin/python -m pytest tests/e2e/test_harness_e2e.py -v --tb=short -m e2e +``` +Expected: FAIL with `ModuleNotFoundError: No module named 'tests.e2e'` (or `fixture 'page' not found`) — the harness does not exist yet. + +- [ ] **Step 4: Implement the harness fixtures** + +`tests/e2e/conftest.py`: +```python +"""Playwright browser E2E fixtures. + +These tests talk to the app through a real headless Chromium browser +against pytest-django's live_server. Seeded data is created in the test +DB (transactionally) and reached over HTTP via live_server.url. +The page fixture holds absolute URLs off live_server; tests may also use +page.goto("/relative/path/") because the page context's base_url is set. +""" + +from __future__ import annotations + +from typing import Iterator + +import pytest +from playwright.sync_api import Browser, Page, sync_playwright + +from django.contrib.auth import get_user_model +from django.utils import timezone +from decimal import Decimal + +User = get_user_model() + +E2E_USERNAME = "e2e_user" +E2E_PASSWORD = "e2e-password-123" + + +@pytest.fixture(scope="session") +def browser() -> Iterator[Browser]: + """Headless Chromium shared across all browser tests in the session.""" + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + yield browser + browser.close() + + +@pytest.fixture() +def page(browser: Browser, live_server) -> Iterator[Page]: + """A fresh browser context bound to pytest-django's live_server.""" + context = browser.new_context(base_url=live_server.url) + page = context.new_page() + yield page + context.close() + + +@pytest.fixture() +def e2e_user(db) -> User: # type: ignore[no-untyped-def] + return User.objects.create_user( + username=E2E_USERNAME, + email="e2e@example.com", + password=E2E_PASSWORD, + ) + + +@pytest.fixture() +def e2e_login(page: Page, e2e_user: User) -> User: # type: ignore[no-untyped-def] + """Log the browser in through the real login form (accounts/login/).""" + page.goto("/accounts/login/") + page.fill('input[name="username"]', E2E_USERNAME) + page.fill('input[name="password"]', E2E_PASSWORD) + page.click('button[type="submit"]') + page.wait_for_url("**/dashboard/") + return e2e_user + + +@pytest.fixture() +def growth_area(db) -> object: # type: ignore[no-untyped-def] + from core.models import GrowthArea + + return GrowthArea.objects.create( + state="TX", + city_name="Austin", + metro_area="Austin-Round Rock", + population_growth_rate=Decimal("0.0214"), + employment_growth_rate=Decimal("0.0341"), + median_income_growth=Decimal("0.0187"), + housing_demand_index=82, + supply_constraint_index=45, + data_timestamp=timezone.now(), + population=978908, + composite_score=Decimal("75.50"), + landlord_score=8, + ) + + +@pytest.fixture() +def discovery_sources(db, growth_area) -> list: # type: ignore[no-untyped-def] + """One record per source in Austin, TX so discovery never hits the network. + + The property_discovery view only triggers background scrapers when a + source table has zero rows for the state; seeding rows short-circuits + that branch and processes the seeded records synchronously. + """ + from core.models import ( + CountyForeclosureNotice, + HudProperty, + UsdaProperty, + VrmProperty, + ) + + now = timezone.now() + vrm = VrmProperty.objects.create( + vrm_property_id=90001, + vrm_listing_url="https://www.vrmproperties.com/property/90001", + address="100 Prime St", + city="Austin", + state="TX", + zip_code="78701", + county="Travis", + list_price=Decimal("200000.00"), + projected_monthly_rent=Decimal("2000.00"), + bedrooms=3, + bathrooms=Decimal("2.0"), + year_built=2001, + status=VrmProperty.Status.FOR_SALE, + scraped_at=now, + last_seen_at=now, + ) + hud = HudProperty.objects.create( + hud_case_number="HUD-482-001", + address="200 Value Ave", + city="Austin", + state="TX", + zip_code="78702", + county="Travis", + asking_price=Decimal("85000.00"), + list_price=Decimal("85000.00"), + bedrooms=3, + bathrooms=Decimal("2.0"), + square_feet=1200, + status=HudProperty.Status.ACTIVE, + scraped_at=now, + last_seen_at=now, + ) + usda = UsdaProperty.objects.create( + usda_case_number="USDA-77-001", + address="300 Rural Ln", + city="Austin", + state="TX", + zip_code="78703", + county="Travis", + list_price=Decimal("150000.00"), + bedrooms=3, + bathrooms=Decimal("2.0"), + square_feet=1500, + status=UsdaProperty.Status.ACTIVE, + scraped_at=now, + last_seen_at=now, + ) + notice = CountyForeclosureNotice.objects.create( + case_number="TC-2026-0001", + document_type=CountyForeclosureNotice.DocumentType.NTS, + address="400 Auction Way", + city="Austin", + state="TX", + zip_code="78704", + county="Travis", + filing_date=timezone.now().date(), + ) + return [vrm, hud, usda, notice] +``` + +Change `test_harness_e2e.py` `test_health_returns_ok` to use the absolute live_server URL: `page.goto("/health/")` works because the context `base_url` is set to `live_server.url` in the harness above, so relative `page.goto` resolves against it. The first smoke test's assertion (`body.inner_text() != ""`) is fine. + +- [ ] **Step 5: Verify the harness tests pass** + +Run: +``` +.venv/bin/python -m pytest tests/e2e/test_harness_e2e.py -v --tb=short -m e2e +``` +Expected: 3 PASS. + +> If `playwright` reports "Executable doesn't exist": run `.venv/bin/python -m playwright install chromium`. + +- [ ] **Step 6: Commit** + +```bash +git add tests/e2e/__init__.py tests/e2e/conftest.py tests/e2e/test_harness_e2e.py +git commit -m "test(e2e): add playwright browser harness and login smoke tests" +``` + +--- + +### Task 2: Full Workflow Journey E2E Test + +**Files:** +- Create: `tests/e2e/test_workflow_e2e.py` + +**Interfaces:** +- Consumes: `e2e_login`, `growth_area`, `discovery_sources` from Task 1; existing pages at `/growth-explorer/`, `/discovery/?growth_area_id=`, `/pipeline/screener/?growth_area_id=`, `/pipeline//offer/`, `/pipeline/list/`; existing view routes `property_discovery` (POST), `pipeline_screener` (POST action=advance), `pipeline_offer_create` (POST). +- Produces: `TestWorkflowJourney` with: + - `test_screener_requires_login` — unauth GET of `/pipeline/screener/` redirects to login. + - `test_full_workflow_journey` — login → growth explorer renders → discovery discovers 4 & passes all → screener shows PASSED → advance to Underwriting → record an offer → pipeline list shows the card. + +- [ ] **Step 1: Write the failing journey test** + +`tests/e2e/test_workflow_e2e.py`: +```python +"""Full-workflow browser E2E: Growth Explorer → Discovery → Screening → Underwriting → Offer → Pipeline.""" + +import re + +import pytest + +pytestmark = pytest.mark.django_db(transaction=True) + + +class TestWorkflowJourney: + def test_screener_requires_login(self, page) -> None: + page.goto("/pipeline/screener/") + assert page.url.endswith("/accounts/login/") + + def test_full_workflow_journey(self, page, e2e_login, growth_area, discovery_sources) -> None: + # ── Growth Explorer renders ──────────────────────────────────── + page.goto("/growth-explorer/") + assert page.locator("h1", has_text="Growth Area Explorer").is_visible() + assert page.locator("#state-select").is_visible() + + # ── Discovery: seeded sources listed, run discovery ──────────── + page.goto(f"/discovery/?growth_area_id={growth_area.pk}") + assert page.locator("#discover-btn").is_visible() + assert page.locator("label", has_text="HUD REO").is_visible() + assert page.locator("label", has_text="VRM (VA REO)").is_visible() + + page.click("#discover-btn") + page.locator("#results-section:not([hidden])").wait_for(state="visible", timeout=30000) + + results_text = page.locator("#results-section").inner_text() + assert "Discovered" in results_text + # All four seeded sources pass screening (see Global Constraints note below). + assert "Passed Screening" in results_text + assert "100 Prime St" in page.inner_text("body") + + # ── Screener: property passed automatic screening ────────────── + page.goto(f"/pipeline/screener/?growth_area_id={growth_area.pk}") + row = page.locator("tr", has_text="100 Prime St") + assert row.is_visible() + assert row.locator(".chip-success", has_text="Passed").is_visible() + + # ── Advance to Underwriting via the screener action ──────────── + row.locator('button', has_text="Underwriting").click() + assert page.locator(".message", has_text="moved to Underwriting").is_visible() + + # Extract the pipeline property pk from the address detail link. + href = page.locator('tr', has_text="100 Prime St").locator('a[href*="/pipeline/"]').first.get_attribute("href") + pk = re.search(r"/pipeline/(\d+)/", href).group(1) + + # ── Offer: record an offer on the property ───────────────────── + page.goto(f"/pipeline/{pk}/offer/") + page.fill('input[name="offer_price"]', "185000") + page.click('button[type="submit"]:has-text("Submit Offer")') + assert page.locator(".message", has_text="Offer recorded.").is_visible() + + # ── Pipeline list: card is present with UNDERWRITING badge ───── + page.goto("/pipeline/list/") + card = page.locator(".pipeline-card", has_text="100 Prime St") + assert card.is_visible() + assert card.locator(".badge-stage", has_text="Underwriting").is_visible() +``` + +> Why all four pass screening: default `ScreeningCriteria` (autocreated by `create_from_*`) allows any state, no property-type/foreclosure filter, no price bounds, `min_gross_yield_pct=7`, `max_price_to_rent_ratio=15`, `min_beds=1`. The VRM yield is 12% (24000/200000) and PTR 8.33; HUD/USDA/county have no rent data so yield/PTR are skipped; all have ≥2 beds. Score stays ≥100, `passed=True`. + +- [ ] **Step 2: Run to verify failure** + +``` +.venv/bin/python -m pytest tests/e2e/test_workflow_e2e.py -v --tb=short -m e2e +``` +Expected: FAIL — discovery results section builds but the journey assertions kick in once the harness (Task 1) exists; run this only after Task 1 lands. First run failure mode is the missing test module (before Task 2 creates it) or a legit assertion failure if the app behavior differs — investigate before moving on. + +- [ ] **Step 3: Implement the minimal wiring (none expected)** + +If run with Task 1 merged, this test should pass against the existing app — the whole point is that the current ALPHA/BETA UI already renders these flows. If any assertion fails, use `--tb=long` and the Playwright trace (`page.screenshot()` inside the test) to find the real gap; fix the test's selectors to match the template, not the template to the test, unless the app truly regressed. + +- [ ] **Step 4: Run to verify pass** + +``` +.venv/bin/python -m pytest tests/e2e/test_workflow_e2e.py -v --tb=short -m e2e +``` +Expected: 2 PASS (login gate + full journey). + +- [ ] **Step 5: Commit** + +```bash +git add tests/e2e/test_workflow_e2e.py +git commit -m "test(e2e): cover full discover->screen->underwrite->offer->pipeline journey" +``` + +--- + +### Task 3: Kanban Drag-and-Drop E2E Test + +**Files:** +- Create: `tests/e2e/test_kanban_e2e.py` + +**Interfaces:** +- Consumes: `e2e_login` from Task 1; `PipelineProperty` model (stage `SCREENING`, status `ACTIVE`); kanban view `/pipeline/kanban/` (GET render + POST `{property_id, new_stage}` returning JSON); kanban template drag handlers on `.kanban-card` (dragstart) and `#col-` (dragover/drop). +- Produces: `TestKanban` with: + - `test_board_renders_stage_columns` — SCREENING and UNDERWRITING columns exist. + - `test_drag_advances_stage` — drag a card from SCREENING to UNDERWRITING, wait for the POST, reload, assert persistence in DOM and DB. + +- [ ] **Step 1: Write the failing kanban tests** + +`tests/e2e/test_kanban_e2e.py`: +```python +"""Browser E2E for the Pipeline Kanban drag-and-drop advance.""" + +import pytest + +from core.models import PipelineProperty + +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.fixture() +def kanban_property(db, e2e_login, growth_area) -> PipelineProperty: + return PipelineProperty.objects.create( + user=e2e_login, + source_type=PipelineProperty.SourceType.HUD, + source_id="KANBAN-0001", + address="101 Boardwalk Blvd", + city="Austin", + state="TX", + zip_code="78701", + county="Travis", + growth_area=growth_area, + stage=PipelineProperty.Stage.SCREENING, + status=PipelineProperty.Status.ACTIVE, + screening_passed=True, + price=90000, + beds=3, + ) + + +class TestKanban: + def test_board_renders_stage_columns(self, page, e2e_login, kanban_property) -> None: + page.goto("/pipeline/kanban/") + assert page.locator('#col-SCREENING .kanban-card[data-id="%d"]' % kanban_property.pk).is_visible() + assert page.locator(".kanban-column", has_text="Underwriting").is_visible() + + def test_drag_advances_stage(self, page, e2e_login, kanban_property) -> None: + page.goto("/pipeline/kanban/") + card_sel = '.kanban-card[data-id="%d"]' % kanban_property.pk + assert page.locator(card_sel).is_visible() + + # Synthetic HTML5 Drag-and-Drop against the column's drop target. + with page.expect_response(lambda r: r.url.endswith("/pipeline/kanban/") and r.request.method == "POST"): + page.evaluate( + """({cardSel, targetSel}) => { + const card = document.querySelector(cardSel); + const target = document.querySelector(targetSel); + const dt = new DataTransfer(); + card.dispatchEvent(new DragEvent('dragstart', {bubbles: true, dataTransfer: dt})); + target.dispatchEvent(new DragEvent('dragover', {bubbles: true, dataTransfer: dt})); + target.dispatchEvent(new DragEvent('drop', {bubbles: true, dataTransfer: dt})); + card.dispatchEvent(new DragEvent('dragend', {bubbles: true, dataTransfer: dt})); + }""", + arg={"cardSel": card_sel, "targetSel": "#col-UNDERWRITING"}, + ) + + # Reload to prove the stage persisted, not just the DOM move. + page.reload() + assert page.locator('#col-UNDERWRITING .kanban-card[data-id="%d"]' % kanban_property.pk).is_visible() + assert page.locator('#col-SCREENING .kanban-card[data-id="%d"]' % kanban_property.pk).count() == 0 + + kanban_property.refresh_from_db() + assert kanban_property.stage == PipelineProperty.Stage.UNDERWRITING +``` + +- [ ] **Step 2: Run to verify failure** + +``` +.venv/bin/python -m pytest tests/e2e/test_kanban_e2e.py -v --tb=short -m e2e +``` +Expected: FAIL — module not found (before this task) or a real behavior assertion failure once the file exists (e.g., the POST rejects backward/unknown stage). In particular `test_drag_advances_stage` should pass if the existing kanban POST works — this test is the regression net the audit wants. + +- [ ] **Step 3: Implement minimal wiring (none expected)** + +The kanban view (`core/views/__init__.py:1504`) already implements the POST handler and the template (`templates/pipeline/kanban.html`) already implements drag/drop — this task only proves it end-to-end. If the drag events don't trigger the handler, check that `DataTransfer` is available in the page's realm (it is in Chromium) and that the fetch fires with `X-CSRFToken` (the template inline `{{ csrf_token }}` supplies it). No app code changes intended. + +- [ ] **Step 4: Run to verify pass** + +``` +.venv/bin/python -m pytest tests/e2e/test_kanban_e2e.py -v --tb=short -m e2e +``` +Expected: 2 PASS. + +- [ ] **Step 5: Commit** + +```bash +git add tests/e2e/test_kanban_e2e.py +git commit -m "test(e2e): cover kanban drag-and-drop stage advance" +``` + +--- + +### Task 4: CI Integration — Run Browser E2E in the Existing Job + +**Files:** +- Modify: `.github/workflows/ci-quality.yml` (the `tests-e2e` job) + +**Interfaces:** +- Consumes: Task 1–3 test files; the existing `tests-e2e` job already runs `pytest tests/ core/tests/ -m e2e` (this recurses into `tests/e2e/`), already installs Playwright + Chromium (`playwright install --with-deps chromium`), and caches browsers. +- Produces: a CI job that serves static assets before the browser tests run, so pages render realistically. + +- [ ] **Step 1: Add collectstatic to the tests-e2e job** + +In `.github/workflows/ci-quality.yml`, inside the `tests-e2e` job, between the "Install Playwright" steps (or "Cache Playwright browsers") and the "Run E2E tests" step, insert: + +```yaml + - name: Collect static files + run: python manage.py collectstatic --noinput --verbosity 0 +``` + +Because Playwright browsers already resolve relative URLs against `live_server.url` (Task 1 harness), no other workflow change is required — the browser tests run inside the same `-m e2e` selection as the existing service-level `_e2e.py` tests. + +- [ ] **Step 2: Verify the workflow renders (dry lint)** + +Run the file through a YAML parse (no structural validation available locally without act): +``` +python -c "import yaml,sys; yaml.safe_load(open('.github/workflows/ci-quality.yml')); print('YAML OK')" +``` +Expected: `YAML OK`. + +- [ ] **Step 3: Run the full e2e selection locally once** + +``` +.venv/bin/python -m pytest tests/ core/tests/ -m e2e -q --tb=short +``` +Expected: existing `test_pipeline_e2e.py` + new `tests/e2e/*.py` all PASS (flaky retry `--reruns 1` applies). Record the count for the CI green run. + +- [ ] **Step 4: Commit** + +```bash +git add .github/workflows/ci-quality.yml +git commit -m "ci: collect static assets before browser e2e runs" +``` + +--- + +## Self-Review + +**Spec coverage (audit Phase 1, line 82–86):** +- Playwright E2E suite for Growth Explorer → Discovery → Screening → Underwriting → Offer → Pipeline → Task 2 (`test_full_workflow_journey`). +- CI integration with headless browser → Task 4 (existing `tests-e2e` job + `collectstatic`; browser runs headless via `chromium.launch(headless=True)`, Task 1). +- Kanban wiring verification (Phase 2's frontend, already present) → Task 3. +- Remaining audit phases (Phase 2 backend endpoints, Phase 3 data-health/screening UX, Phase 4 leasing/comparison) are **out of scope for this plan by design** — they are independent subsystems and should each get their own plan, per the writing-plans scope rule. This plan is the TDD foundation that must land first. + +**Placeholder scan:** No TBD/TODO. Every fixture and assertion contains concrete data and selectors validated against the templates (`screener.html`, `kanban.html`, `offer_form.html`, `property_discovery.html`, `pipeline_list.html`) and views read during plan authoring. + +**Type/signature consistency:** Fixtures named identically in Tasks 1–3: `e2e_login`, `growth_area`, `discovery_sources`, `page`. `PipelineProperty.Stage.UNDERWRITING` used consistently; `#col-` selector matches `kanban.html` (id=`col--{{ col.stage }}`); card `data-id="{{ pp.pk }}"` matches the drag helper. + +**One flagged risk for the implementer:** `test_harness_e2e.py::test_health_returns_ok` asserts the body text contains `ok`. The view `core/views/__init__.py:160` returns `JsonResponse({"status": "ok"})`, which the browser renders as `{"status": "ok"}` — the `in` assertion is correct. If the response shape ever changes, the assertion must follow the view, not the view the assertion. diff --git a/opencode.json b/opencode.json index f32e22c6..ce766c5e 100644 --- a/opencode.json +++ b/opencode.json @@ -1,5 +1,8 @@ { "$schema": "https://opencode.ai/config.json", + "permission": { + "external_directory": "allow" + }, "instructions": ["AGENTS.md"], "skills": { "paths": [".agents/skills", ".agents/roles"] diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..a882d596 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1 @@ +"""Browser-based E2E tests (Playwright).""" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 00000000..0c056b72 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,170 @@ +"""Playwright browser E2E fixtures. + +These tests talk to the app through a real headless Chromium browser +against pytest-django's live_server. Seeded data is created in the test +DB (transactionally) and reached over HTTP via live_server.url. +The page fixture holds absolute URLs off live_server; tests may also use +page.goto("/relative/path/") because the page context's base_url is set. +""" + +from __future__ import annotations + +import os +from typing import Iterator + +# Playwright sync API starts an asyncio event loop internally; Django 6 +# blocks sync DB operations when it detects one. This env-var disables +# that guard for the e2e test process only — acceptable because +# Playwright drives the browser in its own thread and all DB writes +# happen on the pytest main thread. +os.environ.setdefault("DJANGO_ALLOW_ASYNC_UNSAFE", "1") + +import pytest +from playwright.sync_api import Browser, Page, sync_playwright + +from django.contrib.auth import get_user_model +from django.utils import timezone +from decimal import Decimal + +User = get_user_model() + +E2E_USERNAME = "e2e_user" +E2E_PASSWORD = "e2e-password-123" # noqa: S105 — test fixture, not a real secret + + +@pytest.fixture() +def browser() -> Iterator[Browser]: + """Headless Chromium launched per test (function scope avoids finalizer conflicts with reruns).""" + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + yield browser + browser.close() + + +@pytest.fixture() +def page(browser: Browser, live_server) -> Iterator[Page]: + """A fresh browser context bound to pytest-django's live_server.""" + context = browser.new_context(base_url=live_server.url) + page = context.new_page() + yield page + context.close() + + +@pytest.fixture() +def e2e_user(db): # type: ignore[no-untyped-def] + """Create the e2e test user.""" + return User.objects.create_user( + username=E2E_USERNAME, + email="e2e@example.com", + password=E2E_PASSWORD, + ) + + +@pytest.fixture() +def e2e_login(page: Page, e2e_user): # type: ignore[no-untyped-def] + """Log the browser in through the real login form (accounts/login/).""" + page.goto("/accounts/login/") + page.fill('input[name="username"]', E2E_USERNAME) + page.fill('input[name="password"]', E2E_PASSWORD) + page.click('button[type="submit"]') + page.wait_for_url("**/dashboard/") + return e2e_user + + +@pytest.fixture() +def growth_area(db) -> object: # type: ignore[no-untyped-def] + from core.models import GrowthArea + + return GrowthArea.objects.create( + state="TX", + city_name="Austin", + metro_area="Austin-Round Rock", + population_growth_rate=Decimal("0.0214"), + employment_growth_rate=Decimal("0.0341"), + median_income_growth=Decimal("0.0187"), + housing_demand_index=82, + supply_constraint_index=45, + data_timestamp=timezone.now(), + population=978908, + composite_score=Decimal("75.50"), + landlord_score=8, + ) + + +@pytest.fixture() +def discovery_sources(db, growth_area) -> list: # type: ignore[no-untyped-def] + """One record per source in Austin, TX so discovery never hits the network. + + The property_discovery view only triggers background scrapers when a + source table has zero rows for the state; seeding rows short-circuits + that branch and processes the seeded records synchronously. + """ + from core.models import ( + CountyForeclosureNotice, + HudProperty, + UsdaProperty, + VrmProperty, + ) + + now = timezone.now() + vrm = VrmProperty.objects.create( + vrm_property_id=90001, + vrm_listing_url="https://www.vrmproperties.com/property/90001", + address="100 Prime St", + city="Austin", + state="TX", + zip_code="78701", + county="Travis", + list_price=Decimal("200000.00"), + projected_monthly_rent=Decimal("2000.00"), + bedrooms=3, + bathrooms=Decimal("2.0"), + year_built=2001, + status=VrmProperty.Status.FOR_SALE, + scraped_at=now, + last_seen_at=now, + ) + hud = HudProperty.objects.create( + hud_case_number="HUD-482-001", + address="200 Value Ave", + city="Austin", + state="TX", + zip_code="78702", + county="Travis", + asking_price=Decimal("85000.00"), + list_price=Decimal("85000.00"), + bedrooms=3, + bathrooms=Decimal("2.0"), + square_feet=1200, + status=HudProperty.Status.ACTIVE, + scraped_at=now, + last_seen_at=now, + ) + usda = UsdaProperty.objects.create( + usda_case_number="USDA-77-001", + address="300 Rural Ln", + city="Austin", + state="TX", + zip_code="78703", + county="Travis", + list_price=Decimal("150000.00"), + bedrooms=3, + bathrooms=Decimal("2.0"), + square_feet=1500, + status=UsdaProperty.Status.ACTIVE, + scraped_at=now, + last_seen_at=now, + ) + notice = CountyForeclosureNotice.objects.create( + case_number="TC-2026-0001", + document_type=CountyForeclosureNotice.DocumentType.NTS, + address="400 Auction Way", + city="Austin", + state="TX", + zip_code="78704", + county="Travis", + filing_date=timezone.now().date(), + scraped_at=now, + last_seen_at=now, + ) + return [vrm, hud, usda, notice] diff --git a/tests/e2e/test_harness_e2e.py b/tests/e2e/test_harness_e2e.py new file mode 100644 index 00000000..2ba16aa9 --- /dev/null +++ b/tests/e2e/test_harness_e2e.py @@ -0,0 +1,21 @@ +"""Smoke tests proving the Playwright harness binds to live_server and can log in.""" + +import pytest + +pytestmark = [pytest.mark.django_db(transaction=True), pytest.mark.flaky(reruns=0)] + + +class TestHarness: + def test_page_loads_health_check(self, page) -> None: + page.goto("/health/") + assert page.locator("body").inner_text() != "" + + def test_health_returns_ok(self, page) -> None: + page.goto("/health/") + assert "ok" in page.locator("body").inner_text() + + def test_login_redirects_to_dashboard(self, page, e2e_login) -> None: + page.goto("/dashboard/") + assert page.locator("text=Dashboard").count() >= 1 or page.url.endswith( + "/dashboard/" + ) diff --git a/tests/e2e/test_kanban_e2e.py b/tests/e2e/test_kanban_e2e.py new file mode 100644 index 00000000..2229db86 --- /dev/null +++ b/tests/e2e/test_kanban_e2e.py @@ -0,0 +1,75 @@ +"""Browser E2E for the Pipeline Kanban drag-and-drop advance.""" + +import pytest + +from core.models import PipelineProperty + +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.fixture() +def kanban_property(db, e2e_login, growth_area) -> PipelineProperty: + return PipelineProperty.objects.create( + user=e2e_login, + source_type=PipelineProperty.SourceType.HUD, + source_id="KANBAN-0001", + address="101 Boardwalk Blvd", + city="Austin", + state="TX", + zip_code="78701", + county="Travis", + growth_area=growth_area, + stage=PipelineProperty.Stage.SCREENING, + status=PipelineProperty.Status.ACTIVE, + screening_passed=True, + price=90000, + beds=3, + ) + + +class TestKanban: + def test_board_renders_stage_columns( + self, page, e2e_login, kanban_property + ) -> None: + page.goto("/pipeline/kanban/") + assert page.locator( + '#col-SCREENING .kanban-card[data-id="%d"]' % kanban_property.pk + ).is_visible() + assert page.locator(".kanban-column", has_text="Underwriting").is_visible() + + def test_drag_advances_stage(self, page, e2e_login, kanban_property) -> None: + page.goto("/pipeline/kanban/") + card_sel = '.kanban-card[data-id="%d"]' % kanban_property.pk + assert page.locator(card_sel).is_visible() + + # Synthetic HTML5 Drag-and-Drop against the column's drop target. + with page.expect_response( + lambda r: r.url.endswith("/pipeline/kanban/") and r.request.method == "POST" + ): + page.evaluate( + """({cardSel, targetSel}) => { + const card = document.querySelector(cardSel); + const target = document.querySelector(targetSel); + const dt = new DataTransfer(); + card.dispatchEvent(new DragEvent('dragstart', {bubbles: true, dataTransfer: dt})); + target.dispatchEvent(new DragEvent('dragover', {bubbles: true, dataTransfer: dt})); + target.dispatchEvent(new DragEvent('drop', {bubbles: true, dataTransfer: dt})); + card.dispatchEvent(new DragEvent('dragend', {bubbles: true, dataTransfer: dt})); + }""", + arg={"cardSel": card_sel, "targetSel": "#col-UNDERWRITING"}, + ) + + # Reload to prove the stage persisted, not just the DOM move. + page.reload() + assert page.locator( + '#col-UNDERWRITING .kanban-card[data-id="%d"]' % kanban_property.pk + ).is_visible() + assert ( + page.locator( + '#col-SCREENING .kanban-card[data-id="%d"]' % kanban_property.pk + ).count() + == 0 + ) + + kanban_property.refresh_from_db() + assert kanban_property.stage == PipelineProperty.Stage.UNDERWRITING diff --git a/tests/e2e/test_workflow_e2e.py b/tests/e2e/test_workflow_e2e.py new file mode 100644 index 00000000..e3c0aed4 --- /dev/null +++ b/tests/e2e/test_workflow_e2e.py @@ -0,0 +1,71 @@ +"""Full-workflow browser E2E: Growth Explorer → Discovery → Screening → Underwriting → Offer → Pipeline.""" + +import re + +import pytest + +pytestmark = pytest.mark.django_db(transaction=True) + + +class TestWorkflowJourney: + def test_screener_requires_login(self, page) -> None: + page.goto("/pipeline/screener/") + assert "/accounts/login/" in page.url + + def test_full_workflow_journey( + self, page, e2e_login, growth_area, discovery_sources + ) -> None: + # ── Growth Explorer renders ──────────────────────────────────── + page.goto("/growth-explorer/") + assert page.locator("h1", has_text="Growth Area Explorer").is_visible() + assert page.locator("#state-select").is_visible() + + # ── Discovery: seeded sources listed, run discovery ──────────── + page.goto(f"/discovery/?growth_area_id={growth_area.pk}") + assert page.locator("#discover-btn").is_visible() + assert page.locator("label", has_text="HUD REO").is_visible() + assert page.locator("label", has_text="VRM (VA REO)").is_visible() + + page.click("#discover-btn") + page.locator("#results-section:not([hidden])").wait_for( + state="visible", timeout=30000 + ) + + results_text = page.locator("#results-section").inner_text() + assert "Discovered" in results_text + # All four seeded sources pass screening (see Global Constraints note below). + assert "Passed Screening" in results_text + assert "100 Prime St" in page.inner_text("body") + + # ── Screener: property passed automatic screening ────────────── + page.goto(f"/pipeline/screener/?growth_area_id={growth_area.pk}") + row = page.locator("tr", has_text="100 Prime St") + assert row.is_visible() + assert row.locator(".chip-success", has_text="Passed").is_visible() + + # ── Advance to Underwriting via the screener action ──────────── + row.locator("button", has_text="Underwriting").click() + assert page.locator(".message", has_text="moved to Underwriting").is_visible() + + # Extract the pipeline property pk from the address detail link. + href = ( + page.locator("tr", has_text="100 Prime St") + .locator('a[href*="/pipeline/"]') + .first.get_attribute("href") + ) + match = re.search(r"/pipeline/(\d+)/", href) + assert match is not None + pk = match.group(1) + + # ── Offer: record an offer on the property ───────────────────── + page.goto(f"/pipeline/{pk}/offer/") + page.fill('input[name="offer_price"]', "185000") + page.fill('input[name="offer_date"]', "2026-08-20") + page.click('button[type="submit"]:has-text("Submit Offer")') + assert page.locator(".message", has_text="Offer recorded.").is_visible() + + # ── Pipeline list: card is present with UNDERWRITING badge ───── + page.goto("/pipeline/list/") + card = page.locator(".pipeline-card", has_text="100 Prime St") + assert card.is_visible() + assert card.locator(".badge-stage", has_text="Underwriting").is_visible()