-**Mantis** — ticket browser with threat modelling dashboard, disposition scoring, and known malicious IP tracking.
+**Threat Model** — threat modelling dashboard with disposition scoring and known malicious IP tracking.
-
+
+
+
+**Dashboard** — aggregated analytics dashboard.
+
+
| App | What it's for |
|---|---|
| **OpenSearch** | Cross-protocol IP activity matrix, per-protocol drill-down, inline enrichment |
-| **Mantis** | Ticket browser with disposition scoring and known malicious IP tracking |
+| **Threat Model** | Threat modelling dashboard with disposition scoring and known malicious IP tracking |
+| **Dashboard** | Aggregated analytics dashboard |
+| **Mantis Explorer** | Ticket browser and search across the PISCES ticket history |
---
@@ -46,7 +53,7 @@ Four browser-based apps served from a central hub. Launch everything with one co
**Setup**
-| | |
+| Guide | Description |
|---|---|
| [VM Setup](docs/vm-setup.md) | Create an Ubuntu VM and connect to the cyber range network |
| [Getting Started](docs/getting-started.md) | Install, configure, and launch the toolkit on Ubuntu |
@@ -54,7 +61,7 @@ Four browser-based apps served from a central hub. Launch everything with one co
**Using the toolkit**
-| | |
+| Guide | Description |
|---|---|
| [Web UI Workflow](docs/workflow.md) | End-to-end triage walkthrough using the browser-based UI |
| [CLI Workflow](docs/cli-workflow.md) | Terminal-based querier walkthrough — alerts, enrichment, filters, tickets |
@@ -64,7 +71,7 @@ Four browser-based apps served from a central hub. Launch everything with one co
**Reference**
-| | |
+| Guide | Description |
|---|---|
| [Advanced Usage](docs/advanced-usage.md) | Full CLI flag reference for all tools |
| [MCP Server Reference](docs/mcp-servers.md) | Full tool reference for all three MCP servers |
diff --git a/apps/dashboard_web/__init__.py b/apps/dashboard_web/__init__.py
index e69de29..1a14eb6 100644
--- a/apps/dashboard_web/__init__.py
+++ b/apps/dashboard_web/__init__.py
@@ -0,0 +1,18 @@
+"""PISCES Dashboard web application package."""
+
+from datetime import date
+
+
+def safe_date_param(value: str) -> str:
+ """Validate a date query parameter, returning '' for invalid values.
+
+ Prevents user-supplied strings from reaching exception messages
+ (reflected XSS via error rendering).
+ """
+ v = value.strip()
+ if not v:
+ return ""
+ try:
+ return date.fromisoformat(v).isoformat()
+ except ValueError:
+ return ""
diff --git a/apps/dashboard_web/app.py b/apps/dashboard_web/app.py
index 23ec604..8b77813 100644
--- a/apps/dashboard_web/app.py
+++ b/apps/dashboard_web/app.py
@@ -41,11 +41,35 @@ def api_cache_clear():
dcache.invalidate()
return "", 204
+ @app.route("/api/dashboard/sensors")
+ def api_sensor_summary():
+ """Sensor browser modal content — terms agg on host.name."""
+ from apps.dashboard_web.opensearch.aggregations import agg_opensearch_sensors
+
+ time_range = request.args.get("time_range", "now-24h")
+ data = agg_opensearch_sensors(time_range)
+ # Build bucket-like dicts matching the sensor_summary.html template
+ buckets = [
+ {"key": label, "doc_count": count}
+ for label, count in zip(data["labels"], data["counts"])
+ ]
+ current = [
+ s.strip()
+ for s in request.args.get("sensor", "").split(",")
+ if s.strip() and s.strip().lower() != "all"
+ ]
+ return render_template("sensor_summary.html", buckets=buckets, current_sensors=current)
+
from apps.dashboard_web.mantis import bp as mantis_bp
from apps.dashboard_web.opensearch import bp as opensearch_bp
from apps.dashboard_web.overview import bp as overview_bp
+ from apps.dashboard_web.tickets import bp as tickets_bp
- for bp in [overview_bp, opensearch_bp, mantis_bp]:
+ for bp in [overview_bp, opensearch_bp, mantis_bp, tickets_bp]:
app.register_blueprint(bp)
+ from apps.shared.blueprints import make_shared_static_blueprint
+
+ app.register_blueprint(make_shared_static_blueprint())
+
return app
diff --git a/apps/dashboard_web/mantis/__init__.py b/apps/dashboard_web/mantis/__init__.py
index 27b8b40..fc229d8 100644
--- a/apps/dashboard_web/mantis/__init__.py
+++ b/apps/dashboard_web/mantis/__init__.py
@@ -1,9 +1,9 @@
from flask import Blueprint, render_template, request
from apps.dashboard_web import cache as dcache
+from apps.dashboard_web import safe_date_param
from apps.dashboard_web.mantis.aggregations import (
agg_mantis_attack_types,
- agg_mantis_blocklists,
agg_mantis_infra_count,
agg_mantis_timeline,
agg_mantis_top_ips,
@@ -14,20 +14,21 @@
@bp.route("/api/dashboard/mantis")
def section():
- time_range = request.args.get("time_range", "now-24h")
- cached = dcache.get("mantis", {"time_range": time_range})
+ since = safe_date_param(request.args.get("since", ""))
+ until = safe_date_param(request.args.get("until", ""))
+ cache_key = {"since": since, "until": until}
+ cached = dcache.get("mantis", cache_key)
if cached is not None:
return cached
try:
data = {
- "attack_types": agg_mantis_attack_types(),
- "timeline": agg_mantis_timeline(),
- "blocklists": agg_mantis_blocklists(),
- "top_ips": agg_mantis_top_ips(),
+ "attack_types": agg_mantis_attack_types(since, until),
+ "timeline": agg_mantis_timeline(since, until),
+ "top_ips": agg_mantis_top_ips(since, until),
"infra_count": agg_mantis_infra_count(),
}
except Exception as exc:
data = {"error": str(exc)}
- html = render_template("mantis/section.html", data=data, time_range=time_range)
- dcache.put("mantis", {"time_range": time_range}, html)
+ html = render_template("mantis/section.html", data=data)
+ dcache.put("mantis", cache_key, html)
return html
diff --git a/apps/dashboard_web/mantis/aggregations.py b/apps/dashboard_web/mantis/aggregations.py
index 022c45d..985d3e9 100644
--- a/apps/dashboard_web/mantis/aggregations.py
+++ b/apps/dashboard_web/mantis/aggregations.py
@@ -2,22 +2,42 @@
import collections
-from apps.mantis_web.data import INFRA_ROWS, MALICIOUS_ROWS, _raw_tickets
+from apps.threat_model.data import INFRA_ROWS, MALICIOUS_ROWS, _raw_tickets
-def agg_mantis_attack_types() -> list:
+def _filter_tickets(since: str, until: str) -> list:
+ """Filter _raw_tickets by created_at date range."""
+ tickets = _raw_tickets
+ if since:
+ tickets = [t for t in tickets if (t.get("created_at", "") or "") >= since]
+ if until:
+ tickets = [t for t in tickets if (t.get("created_at", "") or "") <= until + "T23:59:59"]
+ return tickets
+
+
+def _filter_malicious(since: str, until: str) -> list:
+ """Filter MALICIOUS_ROWS by last_seen date range."""
+ rows = MALICIOUS_ROWS
+ if since:
+ rows = [r for r in rows if (r.get("last_seen", "") or "") >= since]
+ if until:
+ rows = [r for r in rows if (r.get("first_seen", "") or "") <= until]
+ return rows
+
+
+def agg_mantis_attack_types(since: str = "", until: str = "") -> list:
"""Counter over attack_types in MALICIOUS_ROWS."""
counter = collections.Counter()
- for row in MALICIOUS_ROWS:
+ for row in _filter_malicious(since, until):
for at in row.get("attack_types", []):
counter[at] += 1
return [{"name": k.replace("_", " ").title(), "value": v} for k, v in counter.most_common()]
-def agg_mantis_timeline() -> dict:
+def agg_mantis_timeline(since: str = "", until: str = "") -> dict:
"""Monthly ticket volume from _raw_tickets."""
counter = collections.Counter()
- for t in _raw_tickets:
+ for t in _filter_tickets(since, until):
created = t.get("created_at", "")
if created and len(created) >= 7:
counter[created[:7]] += 1
@@ -25,22 +45,9 @@ def agg_mantis_timeline() -> dict:
return {"months": months, "counts": [counter[m] for m in months]}
-def agg_mantis_blocklists() -> dict:
- """Counter over blocklists in MALICIOUS_ROWS."""
- counter = collections.Counter()
- for row in MALICIOUS_ROWS:
- for bl in row.get("blocklists", []):
- counter[bl] += 1
- items = counter.most_common()
- return {
- "labels": [k for k, _ in items],
- "counts": [v for _, v in items],
- }
-
-
-def agg_mantis_top_ips(n: int = 10) -> list:
+def agg_mantis_top_ips(since: str = "", until: str = "", n: int = 10) -> list:
"""Top N malicious IPs by ticket count."""
- return sorted(MALICIOUS_ROWS, key=lambda r: -r.get("tickets", 0))[:n]
+ return sorted(_filter_malicious(since, until), key=lambda r: -r.get("tickets", 0))[:n]
def agg_mantis_infra_count() -> int:
diff --git a/apps/dashboard_web/mantis/templates/mantis/section.html b/apps/dashboard_web/mantis/templates/mantis/section.html
index 89f1e5e..7cdf13c 100644
--- a/apps/dashboard_web/mantis/templates/mantis/section.html
+++ b/apps/dashboard_web/mantis/templates/mantis/section.html
@@ -3,9 +3,9 @@
{% else %}
-
-PISCES Security Dashboard
+PISCES Toolkit
Select an application to get started
-
-
- PISCES · Settings
+
+
+
+
+
+
+
+
+
+
+
+{% block title %}Mantis Explorer{% endblock title %}
-
-
+
+
+
+
OpenSearch
- Malcolm/Zeek log analysis across 10 protocols. Cross-protocol IP activity matrix, drill-downs, and enrichment.
+
+
+
+
+
+ Tickets Indexed
+ {{ stats.index_age }}
+
+
+ Threat Model Updated
+ {{ stats.model_age }}
+
+
+
+
-
+
+
diff --git a/apps/hub/templates/settings.html b/apps/hub/templates/settings.html
new file mode 100644
index 0000000..47ddcc1
--- /dev/null
+++ b/apps/hub/templates/settings.html
@@ -0,0 +1,155 @@
+
+
+
+
+
+
+
+ OpenSearch
+ Malcolm/Zeek log analysis across 10 protocols. Cross-protocol IP activity matrix, drill-downs, and enrichment.
+
-
-
-
Threat Intel
- Threat modeling and Mantis ticket integration. True positive and false positive IP classification with verdicts.
+
+
+
+
+ Threat Intel
+ Threat modeling and Mantis ticket integration. True positive and false positive IP classification with verdicts.
+
-
-
-
Dashboard
- Analytics and visualizations across all data sources.
+
+
+
+
+ Dashboard
+ Analytics and visualizations across all data sources.
+
-
-
-
Mantis Explorer
- Student activity by institution. Tickets, escalations, and notes with date-range filtering.
+
+
+
+
+ Mantis Explorer
+ Student activity by institution. Tickets, escalations, and notes with date-range filtering.
+
+
+
+Settings
+Preferences are saved locally and apply across all PISCES apps
+
+
+
+
+
+
+
diff --git a/apps/mantis_explorer/app.py b/apps/mantis_explorer/app.py
index 8887d46..c0bfa42 100644
--- a/apps/mantis_explorer/app.py
+++ b/apps/mantis_explorer/app.py
@@ -21,6 +21,10 @@ def create_app() -> Flask:
"""Create and configure the Mantis Explorer Flask app."""
app = Flask(__name__, static_folder="static", template_folder="templates")
+ from apps.shared.blueprints import make_shared_static_blueprint
+
+ app.register_blueprint(make_shared_static_blueprint())
+
@app.context_processor
def inject_globals() -> dict:
return {"script_name": request.environ.get("SCRIPT_NAME", "")}
diff --git a/apps/mantis_explorer/data.py b/apps/mantis_explorer/data.py
index 2071498..982f9ed 100644
--- a/apps/mantis_explorer/data.py
+++ b/apps/mantis_explorer/data.py
@@ -10,12 +10,12 @@
_REPO = os.path.dirname(os.path.dirname(_HERE))
# ---------------------------------------------------------------------------
-# Raw tickets — reuse mantis_web's already-parsed list in hub mode so we
+# Raw tickets — reuse threat_model's already-parsed list in hub mode so we
# don't parse the JSON twice. Falls back to loading independently when
-# running standalone (mantis_web.data will load itself in that process).
+# running standalone (threat_model.data will load itself in that process).
# ---------------------------------------------------------------------------
-from apps.mantis_web.data import TICKETS_BY_ID # noqa: E402, F401
-from apps.mantis_web.data import _raw_tickets as RAW_TICKETS # noqa: E402
+from apps.threat_model.data import TICKETS_BY_ID # noqa: E402, F401
+from apps.threat_model.data import _raw_tickets as RAW_TICKETS # noqa: E402
from src.mantis.activity_report import ( # noqa: E402
StudentStats,
_filter_by_date_range,
diff --git a/apps/mantis_explorer/static/me.css b/apps/mantis_explorer/static/me.css
index 6efa9a7..79dcd6a 100644
--- a/apps/mantis_explorer/static/me.css
+++ b/apps/mantis_explorer/static/me.css
@@ -1,93 +1,16 @@
/* ================================================================
- Mantis Explorer — Material Design 3 Dark/Light Theme
- Self-contained; does not depend on pisces.css
+ Mantis Explorer — App-specific styles
+ Requires: /shared/static/tokens.css, /shared/static/base.css
================================================================ */
-/* ── 1. Design Tokens ─────────────────────────────────── */
-
-[data-theme="dark"] {
- --surface: #0d0f14;
- --surface-container-low: #161a24;
- --surface-container: #1a1f2e;
- --surface-container-high: #1e2433;
- --surface-container-highest:#232840;
- --outline: #2a3045;
- --outline-dim: #1e2233;
- --on-surface: #c8ccd8;
- --on-surface-dim: #5a6278;
- --primary: #4f8ef7;
- --primary-dim: #3a7ae8;
- --secondary: #7ec8e3;
- --tertiary: #bc8cff;
- --green: #3fb950; --yellow: #d29922;
- --red: #f85149; --orange: #e3763c; --purple: #bc8cff;
- --state-hover: rgba(79,142,247,0.08);
- --badge-green-bg: rgba(63,185,80,0.2);
- --badge-red-bg: rgba(248,81,73,0.2);
- --badge-yellow-bg: rgba(210,153,34,0.2);
- --badge-blue-bg: rgba(79,142,247,0.2);
- --badge-gray-bg: rgba(90,98,120,0.3);
- --focus-ring: rgba(79,142,247,0.25);
- --panel-shadow: rgba(0,0,0,0.45);
-}
-
-[data-theme="light"] {
- --surface: #f8f9fc;
- --surface-container-low: #f0f1f5;
- --surface-container: #e8eaef;
- --surface-container-high: #e1e3e9;
- --surface-container-highest:#d8dbe2;
- --outline: #c4c8d4;
- --outline-dim: #dcdfe6;
- --on-surface: #1a1c24;
- --on-surface-dim: #5c6070;
- --primary: #2563eb;
- --primary-dim: #1d4fd8;
- --secondary: #0d7490;
- --tertiary: #7c3aed;
- --green: #16a34a; --yellow: #ca8a04;
- --red: #dc2626; --orange: #ea580c; --purple: #7c3aed;
- --state-hover: rgba(37,99,235,0.06);
- --badge-green-bg: rgba(22,163,74,0.12);
- --badge-red-bg: rgba(220,38,38,0.12);
- --badge-yellow-bg: rgba(202,138,4,0.12);
- --badge-blue-bg: rgba(37,99,235,0.10);
- --badge-gray-bg: rgba(92,96,112,0.12);
- --focus-ring: rgba(37,99,235,0.25);
- --panel-shadow: rgba(0,0,0,0.15);
-}
-
-:root {
- --radius-xs: 4px;
- --radius-sm: 8px;
- --radius-md: 12px;
- --radius-pill: 28px;
-
- --font: "JetBrains Mono","Fira Mono","Courier New",monospace;
- --sans: system-ui,-apple-system,sans-serif;
-}
-
-
-/* ── 2. Reset & Base ──────────────────────────────────── */
-
-*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
-
-html { font-size: 14px; }
+/* ── Body layout ──────────────────────────────────────── */
body {
- background: var(--surface);
- color: var(--on-surface);
- font-family: var(--font);
min-height: 100vh;
display: flex;
flex-direction: column;
- line-height: 1.5;
}
-a { color: var(--primary); text-decoration: none; }
-a:hover { text-decoration: underline; }
-
-
/* ── 3. Nav ───────────────────────────────────────────── */
nav {
@@ -183,6 +106,28 @@ nav .spacer { flex: 1; }
}
.btn-icon:hover { color: var(--primary); text-decoration: none; }
+.btn-notice {
+ background: transparent;
+ border: 1px solid var(--orange);
+ color: var(--orange);
+ font-family: var(--sans);
+ font-size: 0.75rem;
+ font-weight: 700;
+ letter-spacing: 0.05em;
+ text-transform: uppercase;
+ padding: 4px 10px;
+ border-radius: 6px;
+ cursor: pointer;
+ display: inline-flex;
+ align-items: center;
+ gap: 6px;
+ transition: background 0.15s, color 0.15s;
+}
+.btn-notice:hover {
+ background: var(--orange);
+ color: var(--surface);
+}
+
/* ── 4. Main & Page Wrapper ───────────────────────────── */
@@ -261,8 +206,14 @@ main {
transition: border-color 0.15s, box-shadow 0.15s;
color-scheme: dark;
}
-[data-theme="light"] .me-filter-bar input[type="date"],
-[data-theme="light"] .me-filter-bar input[type="text"] {
+[data-theme="pisces-light"] .me-filter-bar input[type="date"],
+[data-theme="pisces-light"] .me-filter-bar input[type="text"],
+[data-theme="gruvbox-light"] .me-filter-bar input[type="date"],
+[data-theme="gruvbox-light"] .me-filter-bar input[type="text"],
+[data-theme="tokyonight-light"] .me-filter-bar input[type="date"],
+[data-theme="tokyonight-light"] .me-filter-bar input[type="text"],
+[data-theme="catppuccin-latte"] .me-filter-bar input[type="date"],
+[data-theme="catppuccin-latte"] .me-filter-bar input[type="text"] {
color-scheme: light;
}
@@ -534,29 +485,6 @@ td.name-cell { max-width: 240px; }
}
-/* ── 12. Buttons ──────────────────────────────────────── */
-
-.btn {
- font-family: var(--sans);
- background: var(--surface-container-high);
- border: 1px solid var(--outline);
- color: var(--on-surface);
- padding: 0 14px;
- border-radius: var(--radius-pill);
- cursor: pointer;
- font-size: 0.82rem;
- height: 30px;
- display: inline-flex;
- align-items: center;
- gap: 6px;
- text-decoration: none;
- transition: border-color 0.15s, color 0.15s;
- white-space: nowrap;
-}
-.btn:hover { border-color: var(--primary); color: var(--primary); text-decoration: none; }
-.btn-sm { padding: 0 10px; font-size: 0.78rem; height: 26px; }
-
-
/* ── 13. Student Slide Panel ──────────────────────────── */
.student-panel-backdrop {
@@ -1060,42 +988,15 @@ tr.sp-ticket-row-active {
box-shadow: inset 3px 0 0 var(--primary);
}
-/* ── 16. HTMX Loading Bar ─────────────────────────────── */
-#htmx-bar {
- position: fixed;
- top: 0; left: 0; right: 0;
- height: 3px;
- z-index: 9999;
- pointer-events: none;
- overflow: hidden;
-}
-#htmx-bar::after {
- content: '';
- display: block;
- height: 100%;
- width: 45%;
- background: var(--primary);
- border-radius: 0 2px 2px 0;
- transform: translateX(-120%);
-}
-#htmx-bar.htmx-bar-active::after {
- animation: htmx-sweep 1.4s ease-in-out infinite;
-}
-#htmx-bar.htmx-bar-done::after {
- animation: none;
- transform: translateX(280%);
- transition: transform 0.25s ease, opacity 0.25s ease;
- opacity: 0;
-}
-@keyframes htmx-sweep {
- 0% { transform: translateX(-120%); }
- 100% { transform: translateX(280%); }
-}
-
-.htmx-indicator { display: none; }
-.htmx-request .htmx-indicator { display: inline; }
+/* ── Spinner ──────────────────────────────────────────── */
+.loading-row td {
+ text-align: center;
+ color: var(--on-surface-dim);
+ padding: 24px;
+ font-style: italic;
+}
/* ── 17. Utilities ────────────────────────────────────── */
@@ -1106,3 +1007,80 @@ tr.sp-ticket-row-active {
padding: 1.5rem;
text-align: center;
}
+
+/* ── 18. Notice Modal ─────────────────────────────────── */
+
+.modal-backdrop {
+ position: fixed;
+ top: 0; left: 0;
+ width: 100vw; height: 100vh;
+ background: rgba(0,0,0,0.8);
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ z-index: 2000;
+ visibility: hidden;
+ opacity: 0;
+ transition: opacity 0.2s ease-in-out;
+ backdrop-filter: blur(4px);
+}
+
+.modal-backdrop.show {
+ visibility: visible;
+ opacity: 1;
+}
+
+.modal-content {
+ background: var(--surface-container-low);
+ border: 1px solid var(--outline);
+ border-top: 3px solid var(--orange);
+ border-radius: var(--radius-md);
+ padding: 1.75rem;
+ max-width: 550px;
+ width: 90%;
+ box-shadow: 0 24px 48px rgba(0,0,0,0.8);
+ display: flex;
+ flex-direction: column;
+ gap: 1.25rem;
+ position: relative;
+}
+
+.modal-header {
+ font-family: var(--sans);
+ font-weight: 700;
+ font-size: 1.1rem;
+ color: var(--orange);
+ display: flex;
+ align-items: center;
+ gap: 12px;
+ text-transform: uppercase;
+ letter-spacing: 0.05em;
+}
+
+.modal-body {
+ font-size: 0.9rem;
+ line-height: 1.6;
+ color: var(--on-surface);
+}
+
+.modal-body p { margin-bottom: 0.75rem; }
+
+.modal-code-box {
+ background: var(--surface);
+ padding: 0.75rem 1rem;
+ border-radius: var(--radius-sm);
+ font-family: var(--font);
+ font-size: 0.78rem;
+ border: 1px solid var(--outline);
+ margin: 0.5rem 0;
+ color: var(--secondary);
+ white-space: nowrap;
+ overflow-x: auto;
+}
+.modal-code-box code { color: inherit; display: inline; }
+
+.modal-footer {
+ display: flex;
+ justify-content: flex-end;
+ margin-top: 0.5rem;
+}
diff --git a/apps/mantis_explorer/static/pisces-logo.ico b/apps/mantis_explorer/static/pisces-logo.ico
deleted file mode 100644
index dd66c23..0000000
Binary files a/apps/mantis_explorer/static/pisces-logo.ico and /dev/null differ
diff --git a/apps/mantis_explorer/static/pisces-logo.png b/apps/mantis_explorer/static/pisces-logo.png
deleted file mode 100644
index 8980d6a..0000000
Binary files a/apps/mantis_explorer/static/pisces-logo.png and /dev/null differ
diff --git a/apps/mantis_explorer/templates/base.html b/apps/mantis_explorer/templates/base.html
index 2ca56bc..e0ec6c9 100644
--- a/apps/mantis_explorer/templates/base.html
+++ b/apps/mantis_explorer/templates/base.html
@@ -1,16 +1,13 @@
-
+
+
+
+
+
+
+ Theme
+ Controls the colour scheme for all apps
+
+
+
+
+
+
+
+
-  }}){kind=logo}
+ 
@@ -63,10 +59,6 @@
@@ -100,29 +93,15 @@
/* ── Notice Modal ─────────────────────────────── */
window.dismissNotice = function() {
document.getElementById('notice-modal').classList.remove('show');
+ sessionStorage.setItem('tmNoticeDismissed', '1');
};
document.addEventListener('DOMContentLoaded', function() {
- document.getElementById('notice-modal').classList.add('show');
+ if (!sessionStorage.getItem('tmNoticeDismissed')) {
+ document.getElementById('notice-modal').classList.add('show');
+ }
});
- /* ── Theme toggle ──────────────────────────────── */
- window.toggleTheme = function() {
- var html = document.documentElement;
- var next = html.getAttribute('data-theme') === 'dark' ? 'light' : 'dark';
- html.setAttribute('data-theme', next);
- localStorage.setItem('pisces-theme', next);
- updateThemeIcon(next);
- };
-
- function updateThemeIcon(theme) {
- var icon = document.getElementById('theme-icon');
- if (!icon) return;
- icon.className = theme === 'dark' ? 'fa-solid fa-moon' : 'fa-solid fa-sun';
- }
-
- updateThemeIcon(document.documentElement.getAttribute('data-theme') || 'dark');
-
/* ── HTMX SCRIPT_NAME prefix (hub mode) ────────── */
document.body.addEventListener('htmx:configRequest', function(evt) {
if (window.SCRIPT_NAME && evt.detail.path.startsWith('/')) {
diff --git a/apps/mantis_web/templates/index.html b/apps/threat_model/templates/index.html
similarity index 100%
rename from apps/mantis_web/templates/index.html
rename to apps/threat_model/templates/index.html
diff --git a/apps/mantis_web/templates/partials/dns_resolver_rows.html b/apps/threat_model/templates/partials/dns_resolver_rows.html
similarity index 100%
rename from apps/mantis_web/templates/partials/dns_resolver_rows.html
rename to apps/threat_model/templates/partials/dns_resolver_rows.html
diff --git a/apps/mantis_web/templates/partials/fp_rows.html b/apps/threat_model/templates/partials/fp_rows.html
similarity index 100%
rename from apps/mantis_web/templates/partials/fp_rows.html
rename to apps/threat_model/templates/partials/fp_rows.html
diff --git a/apps/mantis_web/templates/partials/infra_rows.html b/apps/threat_model/templates/partials/infra_rows.html
similarity index 100%
rename from apps/mantis_web/templates/partials/infra_rows.html
rename to apps/threat_model/templates/partials/infra_rows.html
diff --git a/apps/mantis_web/templates/partials/threat_card.html b/apps/threat_model/templates/partials/threat_card.html
similarity index 100%
rename from apps/mantis_web/templates/partials/threat_card.html
rename to apps/threat_model/templates/partials/threat_card.html
diff --git a/apps/mantis_web/templates/partials/ticket_detail.html b/apps/threat_model/templates/partials/ticket_detail.html
similarity index 100%
rename from apps/mantis_web/templates/partials/ticket_detail.html
rename to apps/threat_model/templates/partials/ticket_detail.html
diff --git a/apps/mantis_web/templates/partials/ticket_list.html b/apps/threat_model/templates/partials/ticket_list.html
similarity index 100%
rename from apps/mantis_web/templates/partials/ticket_list.html
rename to apps/threat_model/templates/partials/ticket_list.html
diff --git a/apps/mantis_web/templates/partials/tp_rows.html b/apps/threat_model/templates/partials/tp_rows.html
similarity index 100%
rename from apps/mantis_web/templates/partials/tp_rows.html
rename to apps/threat_model/templates/partials/tp_rows.html
diff --git a/apps/mantis_web/templates/partials/undetermined_rows.html b/apps/threat_model/templates/partials/undetermined_rows.html
similarity index 100%
rename from apps/mantis_web/templates/partials/undetermined_rows.html
rename to apps/threat_model/templates/partials/undetermined_rows.html
diff --git a/dashboard_web_run.py b/dashboard_web_run.py
deleted file mode 100644
index 97bb20c..0000000
--- a/dashboard_web_run.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-"""Thin shim → apps/dashboard_web/run.py"""
-
-import os
-import runpy
-import sys
-
-_here = os.path.dirname(os.path.abspath(__file__))
-if _here not in sys.path:
- sys.path.insert(0, _here)
-
-runpy.run_path(
- os.path.join(_here, "apps", "dashboard_web", "run.py"),
- run_name="__main__",
-)
diff --git a/docs/advanced-usage.md b/docs/advanced-usage.md
index 0c240ac..0d1709f 100644
--- a/docs/advanced-usage.md
+++ b/docs/advanced-usage.md
@@ -26,14 +26,6 @@ uv run run_all.py --debug
uv run run_all.py --host 127.0.0.1 --port 8080
```
-Or run each app standalone:
-
-```bash
-uv run opensearch_web_run.py # http://0.0.0.0:5001
-uv run mantis_web_run.py # http://0.0.0.0:5003
-uv run dashboard_web_run.py # http://0.0.0.0:5004
-```
-
All launchers accept `--host`, `--port`, and `--debug` flags.
## Standalone Enrichment
diff --git a/docs/assets/pisces-scripts-dashboard-webapp.png b/docs/assets/pisces-scripts-dashboard-webapp.png
new file mode 100644
index 0000000..c2076e9
Binary files /dev/null and b/docs/assets/pisces-scripts-dashboard-webapp.png differ
diff --git a/docs/assets/pisces-scripts-kibana-webapp.png b/docs/assets/pisces-scripts-kibana-webapp.png
deleted file mode 100644
index d4de4ba..0000000
Binary files a/docs/assets/pisces-scripts-kibana-webapp.png and /dev/null differ
diff --git a/docs/assets/pisces-scripts-mantis-webapp.png b/docs/assets/pisces-scripts-mantis-webapp.png
deleted file mode 100644
index 85dca65..0000000
Binary files a/docs/assets/pisces-scripts-mantis-webapp.png and /dev/null differ
diff --git a/docs/assets/pisces-scripts-opensearch-webapp.png b/docs/assets/pisces-scripts-opensearch-webapp.png
index 76c9652..28a69fb 100644
Binary files a/docs/assets/pisces-scripts-opensearch-webapp.png and b/docs/assets/pisces-scripts-opensearch-webapp.png differ
diff --git a/docs/assets/pisces-scripts-threatmodel-one-webapp.png b/docs/assets/pisces-scripts-threatmodel-one-webapp.png
new file mode 100644
index 0000000..466bf7c
Binary files /dev/null and b/docs/assets/pisces-scripts-threatmodel-one-webapp.png differ
diff --git a/docs/assets/pisces-scripts-threatmodel-two-webapp.png b/docs/assets/pisces-scripts-threatmodel-two-webapp.png
new file mode 100644
index 0000000..fdaf45c
Binary files /dev/null and b/docs/assets/pisces-scripts-threatmodel-two-webapp.png differ
diff --git a/mantis_web_run.py b/mantis_web_run.py
deleted file mode 100755
index d1615ef..0000000
--- a/mantis_web_run.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-"""Thin shim → apps/mantis_web/run.py"""
-
-import os
-import runpy
-import sys
-
-_here = os.path.dirname(os.path.abspath(__file__))
-if _here not in sys.path:
- sys.path.insert(0, _here)
-
-runpy.run_path(
- os.path.join(_here, "apps", "mantis_web", "run.py"),
- run_name="__main__",
-)
diff --git a/opensearch_web_run.py b/opensearch_web_run.py
deleted file mode 100755
index e6fdc46..0000000
--- a/opensearch_web_run.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-"""Thin shim → apps/opensearch_web/run.py"""
-
-import os
-import runpy
-import sys
-
-_here = os.path.dirname(os.path.abspath(__file__))
-if _here not in sys.path:
- sys.path.insert(0, _here)
-
-runpy.run_path(
- os.path.join(_here, "apps", "opensearch_web", "run.py"),
- run_name="__main__",
-)
diff --git a/pyproject.toml b/pyproject.toml
index 7767d18..26991a2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "pisces-scripts"
-version = "0.1.0"
+version = "1.0.0"
requires-python = ">=3.12"
dependencies = [
"requests>=2.31.0",
diff --git a/run_all.py b/run_all.py
index 9878952..9becd38 100755
--- a/run_all.py
+++ b/run_all.py
@@ -23,14 +23,14 @@
from apps.dashboard_web.app import create_app as create_dashboard
from apps.hub.app import create_app as create_hub
from apps.mantis_explorer.app import create_app as create_mantis_explorer
-from apps.mantis_web.app import create_app as create_mantis
from apps.opensearch_web.app import create_app as create_opensearch
+from apps.threat_model.app import create_app as create_threat_model
application = DispatcherMiddleware(
create_hub(),
{
"/opensearch": create_opensearch(),
- "/mantis": create_mantis(),
+ "/threat-model": create_threat_model(),
"/dashboard": create_dashboard(),
"/mantis-explorer": create_mantis_explorer(),
},
diff --git a/src/mantis/mantis_index.py b/src/mantis/mantis_index.py
index 07de211..349b3ea 100755
--- a/src/mantis/mantis_index.py
+++ b/src/mantis/mantis_index.py
@@ -10,9 +10,11 @@
"""
import argparse
+import math
import os
import sys
import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
import urllib3
@@ -37,6 +39,39 @@
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
_BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+_FETCH_WORKERS = 8
+
+
+def _fetch_page(
+ session: requests.Session,
+ api_url: str,
+ page: int,
+ page_size: int,
+) -> tuple[int, list[dict] | None]:
+ """Fetch one page from MantisBT with a single retry on timeout.
+
+ Returns (page_number, issues) or (page_number, None) on failure.
+ Thread-safe — called from ThreadPoolExecutor workers.
+ """
+ for attempt in range(2):
+ try:
+ resp = session.get(
+ f"{api_url}/api/rest/issues",
+ params={"page_size": page_size, "page": page},
+ timeout=30,
+ )
+ resp.raise_for_status()
+ return page, resp.json().get("issues", [])
+ except requests.Timeout:
+ if attempt == 0:
+ time.sleep(2)
+ continue
+ console.print(f"[yellow]Page {page} timed out twice — skipping.[/yellow]")
+ return page, None
+ except requests.RequestException as exc:
+ console.print(f"[red]Page {page} failed: {exc}[/red]")
+ return page, None
+ return page, None # unreachable but satisfies type checker
def _fetch_all_raw(
@@ -46,117 +81,122 @@ def _fetch_all_raw(
max_pages: int,
estimated_total: int | None = None,
) -> list[dict]:
- """Phase 1: paginate the MantisBT REST API and collect raw issue dicts."""
- headers = {"Authorization": api_token}
- all_raw: list[dict] = []
-
- # First request to discover total count
- try:
- resp = requests.get(
- f"{api_url}/api/rest/issues",
- headers=headers,
- params={"page_size": page_size, "page": 1},
- timeout=30,
- verify=False,
- )
- resp.raise_for_status()
- except requests.RequestException as exc:
- console.print(f"[red]Initial API request failed: {exc}[/red]")
- return []
+ """Phase 1: paginate the MantisBT REST API and collect raw issue dicts.
- data = resp.json()
- issues = data.get("issues", [])
-
- if not issues:
- console.print("[yellow]API returned no issues on page 1 — nothing to index.[/yellow]")
- return []
+ Fetches page 1 sequentially to discover total_count, then fetches all
+ remaining pages in parallel using a ThreadPoolExecutor. Falls back to
+ sequential pagination when total_count is unavailable and no max_pages cap
+ is set (can't know the page count upfront in that case).
+ """
+ with requests.Session() as session:
+ session.headers.update({"Authorization": api_token})
+ session.verify = False # type: ignore[assignment]
- # Determine best total estimate for the progress bar
- total_known = data.get("total_count") # may be None
- bar_total: int | None = None
- if total_known:
- bar_total = total_known
- console.print(f"[dim]{total_known:,} total tickets reported by API[/dim]")
- elif estimated_total:
- bar_total = estimated_total
- console.print(f"[dim]Estimated ~{estimated_total:,} tickets from existing index[/dim]")
- else:
- console.print("[dim]total_count not available — paginating until empty page[/dim]")
-
- if max_pages:
- bar_total = max_pages * page_size
- console.print(f"[dim]Capped at {max_pages} pages (~{bar_total:,} tickets)[/dim]")
-
- all_raw.extend(issues)
-
- # Build progress columns — use ticket-count display when we have an estimate,
- # otherwise just show count + spinner (no bar/ETA without a total).
- columns = [
- SpinnerColumn(),
- "[progress.description]{task.description}",
- ]
- if bar_total is not None:
- columns += [
- BarColumn(),
- TextColumn("[cyan]{task.completed:,}/{task.total:,} tickets[/cyan]"),
- TimeElapsedColumn(),
- TimeRemainingColumn(),
- ]
- else:
- columns += [
- TextColumn("[cyan]{task.completed:,} tickets[/cyan]"),
- TimeElapsedColumn(),
+ # Probe page 1 to discover total_count and prime the first batch.
+ try:
+ resp = session.get(
+ f"{api_url}/api/rest/issues",
+ params={"page_size": page_size, "page": 1},
+ timeout=30,
+ )
+ resp.raise_for_status()
+ except requests.RequestException as exc:
+ console.print(f"[red]Initial API request failed: {exc}[/red]")
+ return []
+
+ data = resp.json()
+ issues = data.get("issues", [])
+
+ if not issues:
+ console.print("[yellow]API returned no issues on page 1 — nothing to index.[/yellow]")
+ return []
+
+ # Determine best total estimate for the progress bar
+ total_known = data.get("total_count") # may be None
+ bar_total: int | None = None
+ if total_known:
+ bar_total = total_known
+ console.print(f"[dim]{total_known:,} total tickets reported by API[/dim]")
+ elif estimated_total:
+ bar_total = estimated_total
+ console.print(f"[dim]Estimated ~{estimated_total:,} tickets from existing index[/dim]")
+ else:
+ console.print("[dim]total_count not available — paginating until empty page[/dim]")
+
+ if max_pages:
+ bar_total = max_pages * page_size
+ console.print(f"[dim]Capped at {max_pages} pages (~{bar_total:,} tickets)[/dim]")
+
+ all_raw: list[dict] = list(issues)
+
+ # Determine the full set of remaining pages upfront when possible.
+ # Unknown total + no cap → sequential fallback (None sentinel).
+ if total_known:
+ total_pages = math.ceil(total_known / page_size)
+ if max_pages:
+ total_pages = min(total_pages, max_pages)
+ remaining: list[int] | None = list(range(2, total_pages + 1))
+ elif max_pages:
+ remaining = list(range(2, max_pages + 1))
+ else:
+ remaining = None
+
+ # Build progress columns — use ticket-count display when we have an estimate,
+ # otherwise just show count + spinner (no bar/ETA without a total).
+ columns = [
+ SpinnerColumn(),
+ "[progress.description]{task.description}",
]
-
- with Progress(*columns, console=console) as progress:
- task = progress.add_task("Fetching...", total=bar_total)
- progress.advance(task, advance=len(issues))
-
- page = 2
- while True:
- if max_pages and page > max_pages:
- break
-
- retried = False
- while True:
- try:
- resp = requests.get(
- f"{api_url}/api/rest/issues",
- headers=headers,
- params={"page_size": page_size, "page": page},
- timeout=30,
- verify=False,
- )
- resp.raise_for_status()
- break
- except requests.Timeout:
- if not retried:
- retried = True
- console.print(f"[yellow]Timeout on page {page}, retrying...[/yellow]")
- time.sleep(2)
- continue
- console.print(f"[red]Page {page} timed out twice — stopping.[/red]")
- progress.update(task, total=len(all_raw))
- return all_raw
- except requests.RequestException as exc:
- console.print(f"[red]Page {page} failed: {exc}[/red]")
- progress.update(task, total=len(all_raw))
- return all_raw
-
- page_issues = resp.json().get("issues", [])
- if not page_issues:
- break
-
- all_raw.extend(page_issues)
- progress.advance(task, advance=len(page_issues))
-
- if len(page_issues) < page_size:
- break
-
- page += 1
-
- # Snap to exact total now that we know the real count
- progress.update(task, completed=len(all_raw), total=len(all_raw))
+ if bar_total is not None:
+ columns += [
+ BarColumn(),
+ TextColumn("[cyan]{task.completed:,}/{task.total:,} tickets[/cyan]"),
+ TimeElapsedColumn(),
+ TimeRemainingColumn(),
+ ]
+ else:
+ columns += [
+ TextColumn("[cyan]{task.completed:,} tickets[/cyan]"),
+ TimeElapsedColumn(),
+ ]
+
+ with Progress(*columns, console=console) as progress:
+ task = progress.add_task("Fetching...", total=bar_total)
+ progress.advance(task, advance=len(issues))
+
+ if remaining is not None:
+ # Parallel fetch — submit all known pages at once, collect as they finish.
+ page_results: dict[int, list[dict]] = {}
+ with ThreadPoolExecutor(max_workers=_FETCH_WORKERS) as executor:
+ futures = {
+ executor.submit(_fetch_page, session, api_url, p, page_size): p
+ for p in remaining
+ }
+ for future in as_completed(futures):
+ pg, pg_issues = future.result()
+ if pg_issues:
+ page_results[pg] = pg_issues
+ progress.advance(task, advance=len(pg_issues))
+ # Assemble in page order so the index is chronologically stable.
+ for p in sorted(page_results):
+ all_raw.extend(page_results[p])
+ else:
+ # Sequential fallback: total unknown, no page cap.
+ page = 2
+ while True:
+ pg, pg_issues = _fetch_page(session, api_url, page, page_size)
+ if pg_issues is None:
+ progress.update(task, total=len(all_raw))
+ break
+ if not pg_issues:
+ break
+ all_raw.extend(pg_issues)
+ progress.advance(task, advance=len(pg_issues))
+ if len(pg_issues) < page_size:
+ break
+ page += 1
+
+ progress.update(task, completed=len(all_raw), total=len(all_raw))
return all_raw
diff --git a/src/mantis/mantis_search.py b/src/mantis/mantis_search.py
index 841ebd9..2871068 100755
--- a/src/mantis/mantis_search.py
+++ b/src/mantis/mantis_search.py
@@ -391,7 +391,6 @@ def search_via_api(query: str, city: str | None = None, max_pages: int = 10) ->
if not api_url or not api_token:
return []
- headers = {"Authorization": api_token}
query_lower = query.lower()
ip_query = _is_ip_query(query)
ip_re = re.compile(r"\b" + re.escape(query) + r"\b") if ip_query else None
@@ -399,65 +398,67 @@ def search_via_api(query: str, city: str | None = None, max_pages: int = 10) ->
console.print(f"[dim]Querying Mantis REST API for '{query}'...[/dim]")
- for page in range(1, max_pages + 1):
- try:
- resp = requests.get(
- f"{api_url}/api/rest/issues",
- headers=headers,
- params={"page_size": 100, "page": page},
- timeout=20,
- verify=False,
- )
- except requests.RequestException as exc:
- console.print(f"[red]Mantis API request failed: {exc}[/red]")
- break
-
- if resp.status_code == 401:
- console.print("[red]Mantis API auth failed — check MANTIS_API_TOKEN[/red]")
- break
-
- if not resp.ok:
- console.print(f"[red]Mantis API error {resp.status_code}[/red]")
- break
-
- data = resp.json()
- issues = data.get("issues", [])
- if not issues:
- break
-
- for issue in issues:
- text = (
- issue.get("summary", "")
- + " "
- + issue.get("description", "")
- + " "
- + (issue.get("steps_to_reproduce") or "")
- + " "
- + (issue.get("additional_information") or "")
- + " "
- + " ".join(n.get("text", "") for n in issue.get("notes", []))
- )
+ with requests.Session() as session:
+ session.headers.update({"Authorization": api_token})
+ session.verify = False # type: ignore[assignment]
+
+ for page in range(1, max_pages + 1):
+ try:
+ resp = session.get(
+ f"{api_url}/api/rest/issues",
+ params={"page_size": 100, "page": page},
+ timeout=20,
+ )
+ except requests.RequestException as exc:
+ console.print(f"[red]Mantis API request failed: {exc}[/red]")
+ break
+
+ if resp.status_code == 401:
+ console.print("[red]Mantis API auth failed — check MANTIS_API_TOKEN[/red]")
+ break
+
+ if not resp.ok:
+ console.print(f"[red]Mantis API error {resp.status_code}[/red]")
+ break
+
+ data = resp.json()
+ issues = data.get("issues", [])
+ if not issues:
+ break
+
+ for issue in issues:
+ text = (
+ issue.get("summary", "")
+ + " "
+ + issue.get("description", "")
+ + " "
+ + (issue.get("steps_to_reproduce") or "")
+ + " "
+ + (issue.get("additional_information") or "")
+ + " "
+ + " ".join(n.get("text", "") for n in issue.get("notes", []))
+ )
- if ip_query:
- # Word-boundary match to avoid 8.8.8.8 matching 108.8.8.8 or 8.8.8.80
- if not ip_re.search(text):
- continue
- else:
- if query_lower not in text.lower():
- continue
-
- if city:
- project_name = issue.get("project", {}).get("name", "").lower()
- if city.lower() not in project_name:
- continue
-
- results.append(_normalize_issue(issue, api_url))
-
- total = data.get("total_count") or None
- if total is not None and page * 100 >= total:
- break
- if len(issues) < 100:
- break
+ if ip_query:
+ # Word-boundary match to avoid 8.8.8.8 matching 108.8.8.8 or 8.8.8.80
+ if not ip_re.search(text):
+ continue
+ else:
+ if query_lower not in text.lower():
+ continue
+
+ if city:
+ project_name = issue.get("project", {}).get("name", "").lower()
+ if city.lower() not in project_name:
+ continue
+
+ results.append(_normalize_issue(issue, api_url))
+
+ total = data.get("total_count") or None
+ if total is not None and page * 100 >= total:
+ break
+ if len(issues) < 100:
+ break
return results
diff --git a/src/mantis/student_activity.py b/src/mantis/student_activity.py
deleted file mode 100644
index 847f112..0000000
--- a/src/mantis/student_activity.py
+++ /dev/null
@@ -1,794 +0,0 @@
-#!/usr/bin/env python3
-"""
-Student activity report — counts tickets created and notes written per student.
-
-Students are defined as any reporter who has never acted as a ticket handler
-anywhere in the corpus (mirrors the handler_registry logic in mantis_index.py).
-
-Reads from the local offline index by default; pass --live to fetch from the
-REST API instead (requires MANTIS_API_TOKEN + MANTIS_API_URL).
-
-When --student matches exactly one name, a detailed view is shown with ticket
-titles and links. When it matches multiple, the user is prompted to pick one.
-
-Usage:
- python src/mantis/student_activity.py
- python src/mantis/student_activity.py --live
- python src/mantis/student_activity.py --sort tickets
- python src/mantis/student_activity.py --project bonney-lake
- python src/mantis/student_activity.py --student alice
- python src/mantis/student_activity.py --student alice --graph
- python src/mantis/student_activity.py --org 'bellevue college'
- python src/mantis/student_activity.py --org 'bellevue college' --student alice
- python src/mantis/student_activity.py --since 2025-01-01 --until 2025-04-30
- python src/mantis/student_activity.py --input data/tickets/indexed/tickets_index.json
-"""
-
-import argparse
-import json
-import os
-import sys
-from collections import defaultdict
-from dataclasses import dataclass, field
-from datetime import date, timedelta
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
-
-import urllib3
-from dotenv import load_dotenv
-from rich import box
-from rich.console import Console
-from rich.rule import Rule
-from rich.table import Table
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-_BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-DEFAULT_INDEX = os.path.join(_BASE, "data", "tickets", "indexed", "tickets_index.json")
-
-console = Console()
-
-
-def _ordinal(n: int) -> str:
- """Return an integer with its ordinal suffix: 1st, 2nd, 3rd, 4th…"""
- if 11 <= (n % 100) <= 13:
- suffix = "th"
- else:
- suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th")
- return f"{n}{suffix}"
-
-
-def _format_date_range(date_strs: list[str]) -> str:
- """Format a collection of YYYY-MM-DD strings as a human-readable span.
-
- Returns strings like "Jan 1st – Apr 28th" for same-year ranges, or
- "Jan 1st, 2024 – Apr 28th, 2025" when the range crosses calendar years.
- Returns an empty string when no valid dates are present.
- """
- valid = [d for d in date_strs if d and len(d) >= 10]
- if not valid:
- return ""
- parsed = sorted(date.fromisoformat(d[:10]) for d in valid)
- first, last = parsed[0], parsed[-1]
- cross_year = first.year != last.year
-
- def _fmt(d: date) -> str:
- base = f"{d.strftime('%b')} {_ordinal(d.day)}"
- return f"{base}, {d.year}" if cross_year else base
-
- if first == last:
- return _fmt(first)
- return f"{_fmt(first)} – {_fmt(last)}"
-
-
-def _filter_by_date_range(
- tickets: list[dict],
- since: date | None,
- until: date | None,
-) -> list[dict]:
- """Return only tickets whose created_at falls within [since, until].
-
- Tickets with no or unparseable created_at are kept so data is not silently lost.
- """
- if not since and not until:
- return tickets
- result = []
- for t in tickets:
- raw = (t.get("created_at") or "")[:10]
- if not raw:
- result.append(t)
- continue
- try:
- d = date.fromisoformat(raw)
- except ValueError:
- result.append(t)
- continue
- if since and d < since:
- continue
- if until and d > until:
- continue
- result.append(t)
- return result
-
-
-# Minimal ticket ref stored per student — id, summary, url, status, created_at
-_TicketRef = dict
-
-
-@dataclass
-class StudentStats:
- """Activity data for a single student reporter."""
-
- name: str
- created_tickets: list[_TicketRef] = field(default_factory=list)
- # Tickets they left notes on that they did NOT create (keyed by id to deduplicate)
- _noted: dict[str, _TicketRef] = field(default_factory=dict, repr=False)
- notes_written: int = 0
- projects: set[str] = field(default_factory=set)
-
- @property
- def tickets_created(self) -> int:
- return len(self.created_tickets)
-
- @property
- def noted_tickets(self) -> list[_TicketRef]:
- """Unique tickets this student commented on but did not create."""
- return list(self._noted.values())
-
- escalated_tickets: int = 0
- categories: set[str] = field(default_factory=set)
-
- @property
- def total_activity(self) -> int:
- return self.tickets_created + self.notes_written
-
- def add_noted(self, ticket: _TicketRef) -> None:
- """Record a ticket this student left a note on (deduplicates by id)."""
- tid = ticket["id"]
- if tid not in self._noted:
- self._noted[tid] = ticket
-
-
-def _ticket_ref(ticket: dict) -> _TicketRef:
- """Extract the minimal displayable fields from a normalized ticket dict."""
- return {
- "id": ticket.get("id", ""),
- "summary": ticket.get("summary", ""),
- "url": ticket.get("url", ""),
- "status": ticket.get("status", ""),
- "created_at": ticket.get("created_at", ""),
- }
-
-
-def _load_offline(path: str) -> list[dict]:
- """Load normalized tickets from the offline index JSON."""
- if not os.path.exists(path):
- console.print(f"[red]Offline index not found: {path}[/red]")
- console.print("[dim]Run: python src/mantis/mantis_index.py[/dim]")
- sys.exit(1)
- with open(path) as fh:
- return json.load(fh)
-
-
-def _load_live(project_filter: str | None) -> list[dict]:
- """Fetch all tickets from the MantisBT REST API."""
- import requests
-
- from src.mantis.mantis_search import _normalize_issue
-
- api_url = os.environ.get("MANTIS_API_URL", "").rstrip("/")
- api_token = os.environ.get("MANTIS_API_TOKEN", "")
- if not api_url or not api_token:
- console.print("[red]MANTIS_API_URL and MANTIS_API_TOKEN are required for --live[/red]")
- sys.exit(1)
-
- headers = {"Authorization": api_token}
- all_raw: list[dict] = []
- page = 1
-
- console.print("[dim]Fetching tickets from Mantis REST API...[/dim]")
- while True:
- resp = requests.get(
- f"{api_url}/api/rest/issues",
- headers=headers,
- params={"page_size": 200, "page": page},
- timeout=30,
- verify=False,
- )
- if not resp.ok:
- console.print(f"[red]API error {resp.status_code} on page {page}[/red]")
- break
- data = resp.json()
- issues = data.get("issues", [])
- if not issues:
- break
- all_raw.extend(issues)
- total = data.get("total_count")
- if total and page * 200 >= total:
- break
- if len(issues) < 200:
- break
- page += 1
-
- console.print(f"[dim]Fetched {len(all_raw)} tickets — normalizing...[/dim]")
- handler_registry: set[int] = {
- issue["handler"]["id"] for issue in all_raw if issue.get("handler")
- }
- tickets = [_normalize_issue(issue, api_url, handler_registry) for issue in all_raw]
-
- if project_filter:
- tickets = [t for t in tickets if project_filter.lower() in t.get("project", "").lower()]
-
- return tickets
-
-
-def build_report(
- tickets: list[dict],
- project_filter: str | None = None,
-) -> dict[int, StudentStats]:
- """Aggregate per-student activity from a normalized ticket list.
-
- Args:
- tickets: Normalized ticket dicts from the offline index or live API.
- project_filter: Optional substring filter on the project/city field.
-
- Returns:
- Mapping of reporter_id → StudentStats, excluding known handlers.
- """
- if project_filter:
- tickets = [t for t in tickets if project_filter.lower() in t.get("project", "").lower()]
-
- # Rebuild handler registry from this corpus
- handler_ids: set[int] = set()
- for t in tickets:
- h = t.get("handler")
- if h and h.get("id"):
- handler_ids.add(h["id"])
- for note in t.get("notes", []):
- if note.get("is_admin_note"):
- handler_ids.add(note["reporter"]["id"])
-
- stats: dict[int, StudentStats] = defaultdict(lambda: StudentStats(name=""))
-
- for ticket in tickets:
- reporter = ticket.get("reporter", {})
- reporter_id = reporter.get("id", 0)
- reporter_name = reporter.get("name", "unknown")
- project = ticket.get("project", "")
- ref = _ticket_ref(ticket)
-
- if reporter_id not in handler_ids:
- stats[reporter_id].name = reporter_name
- stats[reporter_id].created_tickets.append(ref)
- if project:
- stats[reporter_id].projects.add(project)
- if ticket.get("is_escalated"):
- stats[reporter_id].escalated_tickets += 1
- category = ticket.get("category", "")
- if category:
- stats[reporter_id].categories.add(category)
-
- for note in ticket.get("notes", []):
- note_reporter = note.get("reporter", {})
- note_id = note_reporter.get("id", 0)
- note_name = note_reporter.get("name", "unknown")
-
- if note_id in handler_ids:
- continue
-
- stats[note_id].name = note_name
- stats[note_id].notes_written += 1
- if project:
- stats[note_id].projects.add(project)
- # Only track as "noted" if they didn't create it
- if note_id != reporter_id:
- stats[note_id].add_noted(ref)
-
- return dict(stats)
-
-
-def _pick_student(matches: list[tuple[int, StudentStats]]) -> tuple[int, StudentStats] | None:
- """Prompt the user to select one student from a list of matches."""
- console.print(f"\n[yellow]Found {len(matches)} students matching that name:[/yellow]\n")
- for i, (_, s) in enumerate(matches, 1):
- console.print(
- f" [cyan]{i}[/cyan]. {s.name} "
- f"— {s.tickets_created} ticket(s), {s.notes_written} note(s)"
- )
- console.print()
-
- while True:
- try:
- raw = input("Select a student (number), or press Enter to show all: ").strip()
- except (EOFError, KeyboardInterrupt):
- console.print()
- return None
-
- if raw == "":
- return None
-
- if raw.isdigit() and 1 <= int(raw) <= len(matches):
- return matches[int(raw) - 1]
-
- console.print(f"[red]Enter a number between 1 and {len(matches)}.[/red]")
-
-
-def _pick_org(matches: list[str]) -> str | None:
- """Prompt the user to select one institution from a list of matches."""
- console.print(f"\n[yellow]Found {len(matches)} institutions matching that name:[/yellow]\n")
- for i, name in enumerate(matches, 1):
- console.print(f" [cyan]{i}[/cyan]. {name}")
- console.print()
-
- while True:
- try:
- raw = input("Select an institution (number), or press Enter to show all: ").strip()
- except (EOFError, KeyboardInterrupt):
- console.print()
- return None
-
- if raw == "":
- return None
-
- if raw.isdigit() and 1 <= int(raw) <= len(matches):
- return matches[int(raw) - 1]
-
- console.print(f"[red]Enter a number between 1 and {len(matches)}.[/red]")
-
-
-def display_org_report(
- stats: dict[int, StudentStats],
- org_filter: str,
- sort_by: str = "activity",
- student_filter: str | None = None,
- show_graph: bool = False,
-) -> None:
- """Render the activity report for a single institution.
-
- Finds all unique ticket categories matching org_filter (substring,
- case-insensitive). If multiple categories match, prompts the user to pick
- one or show all. Then renders the same summary table as display_report(),
- scoped to students whose tickets belong to the matched category/categories.
-
- If student_filter is also provided, further narrows to a specific student
- within the institution and shows the detail view.
-
- Args:
- stats: Full per-student stats from build_report().
- org_filter: Substring to match against ticket category names.
- sort_by: Column to sort the summary table by.
- student_filter: Optional student name substring for further drill-down.
- show_graph: Whether to show the submission graph in detail view.
- """
- # Collect all unique categories across the corpus
- all_categories: list[str] = sorted(
- {cat for s in stats.values() for cat in s.categories},
- key=str.lower,
- )
- matching_cats = [c for c in all_categories if org_filter.lower() in c.lower()]
-
- if not matching_cats:
- console.print(f"[yellow]No institution matching '{org_filter}' found.[/yellow]")
- console.print(f"[dim]Known institutions: {', '.join(all_categories) or 'none'}[/dim]")
- return
-
- # Disambiguate if multiple categories match
- if len(matching_cats) == 1:
- chosen_cats = matching_cats
- else:
- chosen = _pick_org(matching_cats)
- chosen_cats = [chosen] if chosen is not None else matching_cats
-
- org_label = chosen_cats[0] if len(chosen_cats) == 1 else org_filter
-
- # Filter students who have tickets in any of the chosen categories
- org_stats = {k: v for k, v in stats.items() if v.categories & set(chosen_cats)}
-
- if not org_stats:
- console.print(f"[yellow]No students found for institution '{org_label}'.[/yellow]")
- return
-
- all_dates = [ref["created_at"] for s in org_stats.values() for ref in s.created_tickets]
- date_range = _format_date_range(all_dates)
- date_part = f" · {date_range}" if date_range else ""
- console.print(
- Rule(f"[bold]{org_label}[/bold] [dim]({len(org_stats)} students{date_part})[/dim]")
- )
-
- if student_filter:
- # Delegate to the normal student-filter path, scoped to this org
- display_report(
- org_stats,
- sort_by=sort_by,
- student_filter=student_filter,
- show_graph=show_graph,
- )
- else:
- display_report(org_stats, sort_by=sort_by)
- if show_graph:
- console.print()
- draw_org_graph(org_stats, org_label)
-
-
-def display_student_detail(student: StudentStats, show_graph: bool = False) -> None:
- """Render a detailed activity breakdown for a single student."""
- console.print(Rule(f"[cyan]{student.name}[/cyan]"))
- escalated_str = (
- f" Escalated: [red]{student.escalated_tickets}[/red]" if student.escalated_tickets else ""
- )
- console.print(
- f" Tickets created: [green]{student.tickets_created}[/green]"
- f"{escalated_str} "
- f"Notes written: [yellow]{student.notes_written}[/yellow] "
- f"Total activity: [bold]{student.total_activity}[/bold]"
- )
- if student.projects:
- console.print(f" Projects: [dim]{', '.join(sorted(student.projects))}[/dim]")
- console.print()
-
- if student.created_tickets:
- created = Table(title="Tickets Created", box=box.SIMPLE, show_header=True)
- created.add_column("#", style="dim", no_wrap=True)
- created.add_column("Summary", ratio=3)
- created.add_column("Status", no_wrap=True)
- created.add_column("Created", no_wrap=True)
- created.add_column("Link", style="blue")
-
- for ref in sorted(
- student.created_tickets,
- key=lambda r: int(r["id"]) if r["id"].isdigit() else 0,
- reverse=True,
- ):
- created.add_row(
- ref["id"],
- ref["summary"] or "—",
- ref["status"] or "—",
- ref["created_at"] or "—",
- ref["url"],
- )
- console.print(created)
-
- noted = student.noted_tickets
- if noted:
- noted_table = Table(
- title="Tickets Commented On (not created by this student)", box=box.SIMPLE
- )
- noted_table.add_column("#", style="dim", no_wrap=True)
- noted_table.add_column("Summary", ratio=3)
- noted_table.add_column("Status", no_wrap=True)
- noted_table.add_column("Link", style="blue")
-
- for ref in sorted(
- noted, key=lambda r: int(r["id"]) if r["id"].isdigit() else 0, reverse=True
- ):
- noted_table.add_row(
- ref["id"],
- ref["summary"] or "—",
- ref["status"] or "—",
- ref["url"],
- )
- console.print(noted_table)
-
- if show_graph:
- console.print()
- draw_submission_graph(student)
-
-
-def _plot_ticket_timeline(date_strs: list[str], title: str) -> None:
- """Render a terminal line graph of ticket submissions over time.
-
- Accepts a flat list of YYYY-MM-DD strings (duplicates allowed — each
- represents one ticket). Granularity is chosen automatically:
- - ≤ 5 weeks → daily
- - ≤ 12 months → weekly (Mon-anchored)
- - > 12 months → monthly
-
- The graph width is capped to the terminal width minus a small margin.
- """
- import plotext as plt
-
- parsed = sorted(date.fromisoformat(d) for d in date_strs)
- span_days = (parsed[-1] - parsed[0]).days
-
- if span_days <= 35:
- granularity = "day"
- label_fmt = "%b %d"
- bucket_fn = lambda d: d # noqa: E731
- elif span_days <= 365:
- granularity = "week"
- label_fmt = "%b %d"
- bucket_fn = lambda d: d - timedelta(days=d.weekday()) # noqa: E731
- else:
- granularity = "month"
- label_fmt = "%b '%y"
- bucket_fn = lambda d: d.replace(day=1) # noqa: E731
-
- buckets: dict[date, int] = defaultdict(int)
- for d in parsed:
- buckets[bucket_fn(d)] += 1
-
- # Fill gaps so the x-axis is contiguous
- all_buckets = sorted(buckets)
- first, last = all_buckets[0], all_buckets[-1]
- if granularity == "day":
- cursor, step = first, timedelta(days=1)
- full_range: list[date] = []
- while cursor <= last:
- full_range.append(cursor)
- cursor += step
- elif granularity == "week":
- cursor, step = first, timedelta(weeks=1)
- full_range = []
- while cursor <= last:
- full_range.append(cursor)
- cursor += step
- else:
- full_range = []
- y, m = first.year, first.month
- while date(y, m, 1) <= last:
- full_range.append(date(y, m, 1))
- m += 1
- if m > 12:
- m, y = 1, y + 1
-
- counts = [buckets.get(b, 0) for b in full_range]
- labels = [b.strftime(label_fmt) for b in full_range]
- # Use numeric x indices — passing formatted strings causes plotext to attempt
- # date parsing which fails for short formats like "Jan 12".
- x_indices = list(range(len(counts)))
-
- try:
- term_width = os.get_terminal_size().columns
- except OSError:
- term_width = 100
- plot_width = max(60, min(term_width - 4, 160))
-
- tick_step = max(1, len(labels) // (plot_width // 10))
- tick_positions = x_indices[::tick_step]
- tick_labels = labels[::tick_step]
-
- plt.clf()
- plt.plot_size(plot_width, 14)
- plt.theme("dark")
- plt.title(f"{title} ({granularity}ly)")
- plt.xlabel(granularity.capitalize())
- plt.ylabel("Tickets")
- plt.plot(x_indices, counts, marker="braille")
- plt.xticks(tick_positions, tick_labels)
- plt.show()
- console.print(
- f" [dim]{len(parsed)} ticket(s) · "
- f"{full_range[0].strftime('%Y-%m-%d')} → {full_range[-1].strftime('%Y-%m-%d')} "
- f"· {granularity}ly buckets[/dim]"
- )
-
-
-def draw_submission_graph(student: StudentStats) -> None:
- """Render a submission timeline graph for a single student."""
- dated = [
- ref["created_at"]
- for ref in student.created_tickets
- if ref.get("created_at") and len(ref["created_at"]) == 10
- ]
- if not dated:
- console.print("[yellow]No dated tickets — cannot draw graph.[/yellow]")
- return
- _plot_ticket_timeline(dated, f"Ticket Submissions — {student.name}")
-
-
-def draw_org_graph(org_stats: dict[int, StudentStats], org_label: str) -> None:
- """Render an aggregate submission timeline graph for all students in an org."""
- dated = [
- ref["created_at"]
- for s in org_stats.values()
- for ref in s.created_tickets
- if ref.get("created_at") and len(ref["created_at"]) == 10
- ]
- if not dated:
- console.print("[yellow]No dated tickets — cannot draw graph.[/yellow]")
- return
- _plot_ticket_timeline(dated, f"Ticket Submissions — {org_label}")
-
-
-def display_report(
- stats: dict[int, StudentStats],
- sort_by: str = "activity",
- student_filter: str | None = None,
- org_filter: str | None = None,
- show_graph: bool = False,
-) -> None:
- """Render the student activity report.
-
- If org_filter is set, delegates to display_org_report() which scopes the
- table to students from that institution (matched by ticket category).
- If student_filter matches exactly one student, shows the detail view.
- If it matches multiple, prompts the user to pick one (or show all).
- With no filter, renders the full summary table.
- """
- if org_filter:
- display_org_report(
- stats,
- org_filter=org_filter,
- sort_by=sort_by,
- student_filter=student_filter,
- show_graph=show_graph,
- )
- return
-
- if student_filter:
- matches = [(k, v) for k, v in stats.items() if student_filter.lower() in v.name.lower()]
-
- if not matches:
- console.print(f"[yellow]No student matching '{student_filter}' found.[/yellow]")
- return
-
- if len(matches) == 1:
- display_student_detail(matches[0][1], show_graph=show_graph)
- return
-
- # Multiple matches — prompt
- chosen = _pick_student(matches)
- if chosen is not None:
- display_student_detail(chosen[1], show_graph=show_graph)
- return
-
- # User pressed Enter — fall through and show all matches in summary table
- stats = dict(matches)
-
- rows = list(stats.values())
-
- if sort_by == "tickets":
- rows.sort(key=lambda s: s.tickets_created, reverse=True)
- elif sort_by == "notes":
- rows.sort(key=lambda s: s.notes_written, reverse=True)
- elif sort_by == "name":
- rows.sort(key=lambda s: s.name.lower())
- else:
- rows.sort(key=lambda s: s.total_activity, reverse=True)
-
- all_dates = [ref["created_at"] for s in rows for ref in s.created_tickets]
- date_range = _format_date_range(all_dates)
- title = f"Student Activity Report ({len(rows)} students)"
- if date_range:
- title += f" · {date_range}"
-
- table = Table(
- title=title,
- box=box.SIMPLE,
- show_footer=True,
- )
- table.add_column("Student", style="cyan", footer="TOTAL")
- table.add_column(
- "Tickets Created",
- justify="right",
- style="green",
- footer=str(sum(s.tickets_created for s in rows)),
- )
- table.add_column(
- "Escalated",
- justify="right",
- style="red",
- footer=str(sum(s.escalated_tickets for s in rows)),
- )
- table.add_column(
- "Notes Written",
- justify="right",
- style="yellow",
- footer=str(sum(s.notes_written for s in rows)),
- )
- table.add_column(
- "Total Activity",
- justify="right",
- footer=str(sum(s.total_activity for s in rows)),
- )
- table.add_column("Projects", style="dim")
-
- for s in rows:
- projects_str = ", ".join(sorted(s.projects)) if s.projects else "—"
- escalated_str = str(s.escalated_tickets) if s.escalated_tickets else "—"
- table.add_row(
- s.name,
- str(s.tickets_created),
- escalated_str,
- str(s.notes_written),
- str(s.total_activity),
- projects_str,
- )
-
- console.print(table)
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(description="PISCES Student Activity Report")
- parser.add_argument(
- "--input",
- default=DEFAULT_INDEX,
- help="Path to tickets_index.json (default: data/tickets/indexed/tickets_index.json)",
- )
- parser.add_argument(
- "--live",
- action="store_true",
- help="Fetch from Mantis REST API instead of the offline index",
- )
- parser.add_argument(
- "--project",
- metavar="NAME",
- help="Filter to tickets in projects matching NAME (substring, e.g. bonney-lake)",
- )
- parser.add_argument(
- "--sort",
- choices=["activity", "tickets", "notes", "name"],
- default="activity",
- help="Sort order for the summary table (default: activity)",
- )
- parser.add_argument(
- "--student",
- metavar="NAME",
- help="Filter to a specific student by name (substring, case-insensitive)",
- )
- parser.add_argument(
- "--org",
- metavar="NAME",
- help="Filter to students from a specific institution by category name (substring, "
- "case-insensitive, e.g. 'bellevue college')",
- )
- parser.add_argument(
- "--since",
- metavar="YYYY-MM-DD",
- help="Include only tickets created on or after this date",
- )
- parser.add_argument(
- "--until",
- metavar="YYYY-MM-DD",
- help="Include only tickets created on or before this date",
- )
- parser.add_argument(
- "--graph",
- action="store_true",
- help="Show a terminal line graph of ticket submissions over time "
- "(requires --student or --org)",
- )
- args = parser.parse_args()
-
- if args.graph and not args.student and not args.org:
- parser.error("--graph requires --student or --org")
-
- since: date | None = None
- until: date | None = None
- if args.since:
- try:
- since = date.fromisoformat(args.since)
- except ValueError:
- parser.error(f"--since: invalid date '{args.since}' (expected YYYY-MM-DD)")
- if args.until:
- try:
- until = date.fromisoformat(args.until)
- except ValueError:
- parser.error(f"--until: invalid date '{args.until}' (expected YYYY-MM-DD)")
-
- load_dotenv()
-
- from src.utils.dns import setup_dns
-
- setup_dns()
-
- if args.live:
- tickets = _load_live(project_filter=args.project)
- stats = build_report(_filter_by_date_range(tickets, since, until))
- else:
- tickets = _load_offline(args.input)
- stats = build_report(
- _filter_by_date_range(tickets, since, until), project_filter=args.project
- )
-
- display_report(
- stats,
- sort_by=args.sort,
- student_filter=args.student,
- org_filter=args.org,
- show_graph=args.graph,
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/src/querier/filter_loader.py b/src/querier/filter_loader.py
index 78c0f86..8ba3914 100755
--- a/src/querier/filter_loader.py
+++ b/src/querier/filter_loader.py
@@ -11,6 +11,24 @@
import yaml
+# Module-level cache: (filters_dir, municipality) → {"mtime": float, "result": dict}
+_filter_cache: dict[tuple, dict] = {}
+
+
+def _max_mtime(filters_dir: str) -> float:
+ """Return the highest mtime across all *.yaml files under filters_dir."""
+ max_t = 0.0
+ for root, _dirs, files in os.walk(filters_dir):
+ for fname in files:
+ if fname.endswith(".yaml") or fname.endswith(".yml"):
+ try:
+ t = os.path.getmtime(os.path.join(root, fname))
+ if t > max_t:
+ max_t = t
+ except OSError:
+ pass
+ return max_t
+
def load_filters(
filters_dir: str,
@@ -30,6 +48,12 @@ def load_filters(
"errors": [str, ...] # parse/schema errors
}
"""
+ cache_key = (filters_dir, municipality)
+ current_mtime = _max_mtime(filters_dir)
+ cached = _filter_cache.get(cache_key)
+ if cached is not None and cached["mtime"] == current_mtime:
+ return cached["result"]
+
must_not_clauses: list[dict] = []
errors: list[str] = []
filter_count = 0
@@ -88,11 +112,13 @@ def load_filters(
must_not_clauses.append(entry)
filter_count += 1
- return {
+ result = {
"must_not": must_not_clauses,
"filter_count": filter_count,
"errors": errors,
}
+ _filter_cache[cache_key] = {"mtime": current_mtime, "result": result}
+ return result
if __name__ == "__main__":
diff --git a/src/querier/opensearch_querier.py b/src/querier/opensearch_querier.py
index a5e9373..fabd4ad 100755
--- a/src/querier/opensearch_querier.py
+++ b/src/querier/opensearch_querier.py
@@ -128,6 +128,11 @@ def _build_parser(module) -> argparse.ArgumentParser:
action="store_true",
help="Use cached OpenSearch response if available",
)
+ parser.add_argument(
+ "--profile",
+ action="store_true",
+ help="Add profile:true to the ES query and display per-shard timing after results",
+ )
# Protocol-specific args from the module
module.add_args(parser)
@@ -185,7 +190,7 @@ def main() -> None:
if args.sensor and args.sensor.lower() != "all":
sensors = [s.strip() for s in args.sensor.split(",")]
- extra_must = module.build_extra_must(search_params)
+ extra_must, _ = module.build_extra_must(search_params)
body, _ = build_base_query(
must_not=must_not,
extra_must=extra_must,
diff --git a/src/querier/zeek_modules/base.py b/src/querier/zeek_modules/base.py
index aa3bfc3..295c95f 100644
--- a/src/querier/zeek_modules/base.py
+++ b/src/querier/zeek_modules/base.py
@@ -43,6 +43,10 @@
# Constants
# ---------------------------------------------------------------------------
+# Fetch this many times the requested limit when post-filters are active so
+# truncation still yields enough rows after Python-side filtering discards hits.
+_OVERFETCH_MULTIPLIER = 3
+
OPENSEARCH_URL = os.environ.get("OPENSEARCH_URL", "https://pisces-opensearch.cyberrangepoulsbo.com")
INDEX = "arkime_sessions3-*"
@@ -232,6 +236,90 @@ def query_opensearch(body: dict, params: dict) -> dict | None:
return resp.json()
+# ---------------------------------------------------------------------------
+# Profile display
+# ---------------------------------------------------------------------------
+
+
+def _walk_clauses(node: dict, acc: list) -> None:
+ """DFS walk of an ES profile query node, collecting (type, description, time_ms)."""
+ if not isinstance(node, dict):
+ return
+ time_ms = node.get("time_in_nanos", 0) / 1_000_000
+ acc.append((node.get("type", ""), node.get("description", ""), time_ms))
+ for child in node.get("children", []):
+ _walk_clauses(child, acc)
+
+
+def display_profile(raw: dict) -> None:
+ """Render OpenSearch profile data: per-shard timing and slowest query clauses."""
+ profile = raw.get("profile")
+ if not profile:
+ console.print("[yellow]No profile data in response.[/yellow]")
+ return
+
+ shards = profile.get("shards", [])
+ if not shards:
+ console.print("[yellow]Profile present but no shard data.[/yellow]")
+ return
+
+ # --- Shard summary table ---
+ shard_table = Table(title="Profile — per-shard timing", box=box.SIMPLE_HEAVY)
+ shard_table.add_column("Shard", style="cyan", no_wrap=True)
+ shard_table.add_column("Query (ms)", justify="right")
+ shard_table.add_column("Fetch (ms)", justify="right")
+
+ total_query_ms = 0.0
+ total_fetch_ms = 0.0
+ all_clauses: list = []
+
+ for shard in shards:
+ shard_id = shard.get("id", "?")
+ query_ms = 0.0
+ fetch_ms = 0.0
+
+ for search in shard.get("searches", []):
+ for q in search.get("query", []):
+ query_ms += q.get("time_in_nanos", 0) / 1_000_000
+ _walk_clauses(q, all_clauses)
+ for collector in search.get("collector", []):
+ fetch_ms += collector.get("time_in_nanos", 0) / 1_000_000
+
+ total_query_ms += query_ms
+ total_fetch_ms += fetch_ms
+ shard_table.add_row(shard_id, f"{query_ms:.2f}", f"{fetch_ms:.2f}")
+
+ shard_table.add_section()
+ shard_table.add_row(
+ "[bold]Total[/bold]",
+ f"[bold]{total_query_ms:.2f}[/bold]",
+ f"[bold]{total_fetch_ms:.2f}[/bold]",
+ )
+ console.print(shard_table)
+
+ # --- Slowest clauses table (top 15, aggregated across shards) ---
+ from collections import defaultdict
+
+ clause_totals: dict = defaultdict(float)
+ clause_types: dict = {}
+ for ctype, desc, ms in all_clauses:
+ key = desc or ctype
+ clause_totals[key] += ms
+ clause_types[key] = ctype
+
+ top = sorted(clause_totals.items(), key=lambda kv: -kv[1])[:15]
+ if top:
+ clause_table = Table(
+ title="Profile — slowest clauses (all shards combined)", box=box.SIMPLE_HEAVY
+ )
+ clause_table.add_column("Type", style="dim", no_wrap=True)
+ clause_table.add_column("Description", overflow="fold")
+ clause_table.add_column("Total (ms)", justify="right")
+ for desc, ms in top:
+ clause_table.add_row(clause_types.get(desc, ""), desc, f"{ms:.2f}")
+ console.print(clause_table)
+
+
# ---------------------------------------------------------------------------
# Filter loading
# ---------------------------------------------------------------------------
@@ -308,7 +396,7 @@ def build_base_query(
"sort": [{"@timestamp": {"order": "desc"}}],
"query": {
"bool": {
- "must": must_clauses,
+ "filter": must_clauses,
"must_not": effective_must_not,
}
},
@@ -316,7 +404,7 @@ def build_base_query(
}
params = {
- "path": f"{INDEX}/_search",
+ "path": f"{INDEX}/_search?timeout=30s",
"method": "POST",
}
@@ -383,7 +471,9 @@ def run_query(module, search_params: dict) -> list:
# Over-fetch when post-filters are active so truncation still yields enough rows.
requested_limit = search_params.get("limit", 500)
- query_limit = min(requested_limit * 3, 5000) if post_filters else requested_limit
+ query_limit = (
+ min(requested_limit * _OVERFETCH_MULTIPLIER, 5000) if post_filters else requested_limit
+ )
body, params = build_base_query(
must_not=must_not,
@@ -401,8 +491,11 @@ def run_query(module, search_params: dict) -> list:
time_to=search_params.get("time_to"),
)
- # Cache handling
- use_cache = search_params.get("use_cache", False)
+ if search_params.get("profile"):
+ body["profile"] = True
+
+ # Cache is meaningless for profile runs (timing data is request-specific).
+ use_cache = search_params.get("use_cache", False) and not search_params.get("profile")
raw = None
cache_key = hashlib.md5(json.dumps(body, sort_keys=True).encode()).hexdigest()[:10]
cpath = _cache_path(cache_key)
@@ -422,7 +515,11 @@ def run_query(module, search_params: dict) -> list:
if search_params.get("raise_on_error"):
raise RuntimeError("OpenSearch query failed — check credentials and OPENSEARCH_URL")
return []
- _save_cache(raw, cpath)
+ if not search_params.get("profile"):
+ _save_cache(raw, cpath)
+
+ if search_params.get("profile"):
+ display_profile(raw)
hits = raw.get("hits", {}).get("hits", [])
if not hits:
@@ -650,7 +747,7 @@ def list_sensors(time_range: str = "now-7d") -> None:
"size": 0,
"query": {
"bool": {
- "must": [
+ "filter": [
{"range": {"@timestamp": {"gte": time_range, "lte": "now"}}},
{"exists": {"field": "event.dataset"}},
]
@@ -701,7 +798,7 @@ def list_log_types(time_range: str = "now-7d") -> None:
"size": 0,
"query": {
"bool": {
- "must": [
+ "filter": [
{"range": {"@timestamp": {"gte": time_range, "lte": "now"}}},
{"exists": {"field": "event.dataset"}},
]
diff --git a/tests/test_dashboard_date_param.py b/tests/test_dashboard_date_param.py
new file mode 100644
index 0000000..363de3c
--- /dev/null
+++ b/tests/test_dashboard_date_param.py
@@ -0,0 +1,27 @@
+"""Regression tests for dashboard date parameter sanitisation (reflected XSS)."""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import pytest
+
+from apps.dashboard_web import safe_date_param
+
+
+@pytest.mark.parametrize(
+ "raw, expected",
+ [
+ ("2026-01-15", "2026-01-15"),
+ (" 2026-01-15 ", "2026-01-15"),
+ ("", ""),
+ (" ", ""),
+ ("", ""),
+ ("2026-99-99", ""),
+ ("not-a-date", ""),
+ ("2026-01-15; DROP TABLE", ""),
+ ],
+)
+def test_safe_date_param(raw: str, expected: str) -> None:
+ assert safe_date_param(raw) == expected
diff --git a/tests/test_flask_apps.py b/tests/test_flask_apps.py
index de9350c..ac1c0e3 100644
--- a/tests/test_flask_apps.py
+++ b/tests/test_flask_apps.py
@@ -3,7 +3,7 @@
These tests verify that each app can be created and that basic routes respond
correctly, without requiring a running OpenSearch or Mantis backend.
-Data-dependent apps (mantis_web, opensearch_web, dashboard_web) have their
+Data-dependent apps (threat_model, opensearch_web, dashboard_web) have their
data loading mocked so the tests remain fast and offline.
"""
@@ -50,7 +50,7 @@ def test_hub_index_contains_app_links() -> None:
html = resp.data.decode()
# Hub should link to the other apps
assert "OpenSearch" in html or "opensearch" in html.lower()
- assert "Mantis" in html or "mantis" in html.lower()
+ assert "Mantis" in html or "threat" in html.lower()
# ---------------------------------------------------------------------------
@@ -91,7 +91,7 @@ def test_opensearch_overview_route_exists() -> None:
def _make_mock_data() -> MagicMock:
- """Build a MagicMock that satisfies all apps.mantis_web.data imports."""
+ """Build a MagicMock that satisfies all apps.threat_model.data imports."""
m = MagicMock()
m.MALICIOUS_ROWS = []
m.FP_ROWS = []
@@ -113,26 +113,26 @@ def _make_mock_data() -> MagicMock:
return m
-def test_mantis_web_create_app_returns_flask() -> None:
+def test_threat_model_create_app_returns_flask() -> None:
from flask import Flask
mock_data = _make_mock_data()
- with patch.dict(sys.modules, {"apps.mantis_web.data": mock_data}):
+ with patch.dict(sys.modules, {"apps.threat_model.data": mock_data}):
# Re-import to pick up the mock
- if "apps.mantis_web.app" in sys.modules:
- del sys.modules["apps.mantis_web.app"]
- from apps.mantis_web.app import create_app
+ if "apps.threat_model.app" in sys.modules:
+ del sys.modules["apps.threat_model.app"]
+ from apps.threat_model.app import create_app
app = create_app()
assert isinstance(app, Flask)
-def test_mantis_web_routes_registered() -> None:
+def test_threat_model_routes_registered() -> None:
mock_data = _make_mock_data()
- with patch.dict(sys.modules, {"apps.mantis_web.data": mock_data}):
- if "apps.mantis_web.app" in sys.modules:
- del sys.modules["apps.mantis_web.app"]
- from apps.mantis_web.app import create_app
+ with patch.dict(sys.modules, {"apps.threat_model.data": mock_data}):
+ if "apps.threat_model.app" in sys.modules:
+ del sys.modules["apps.threat_model.app"]
+ from apps.threat_model.app import create_app
app = create_app()
rules = {rule.rule for rule in app.url_map.iter_rules()}
diff --git a/tests/test_mantis_web_helpers.py b/tests/test_threat_model_helpers.py
similarity index 68%
rename from tests/test_mantis_web_helpers.py
rename to tests/test_threat_model_helpers.py
index ca71d68..d27dd38 100644
--- a/tests/test_mantis_web_helpers.py
+++ b/tests/test_threat_model_helpers.py
@@ -1,4 +1,4 @@
-"""Tests for pure helper functions in apps/mantis_web/.
+"""Tests for pure helper functions in apps/threat_model/.
Covers:
- data.py pure functions: fmt_attack, country_flag, days_between, _malicious_row
@@ -22,16 +22,16 @@
def _import_data_with_empty_files():
- """Import apps.mantis_web.data with all file loading stubbed out."""
+ """Import apps.threat_model.data with all file loading stubbed out."""
# Remove cached module so the patched version is freshly imported
for key in list(sys.modules):
- if "apps.mantis_web" in key:
+ if "apps.threat_model" in key:
del sys.modules[key]
with patch("builtins.open", MagicMock()):
with patch("json.load", return_value=[]):
- import apps.mantis_web.data as data_mod
+ import apps.threat_model.data as data_mod
return data_mod
@@ -43,17 +43,17 @@ def _import_data_with_empty_files():
class TestFmtAttack:
def test_underscore_to_space(self) -> None:
- from apps.mantis_web.data import fmt_attack
+ from apps.threat_model.data import fmt_attack
assert fmt_attack("port_scan") == "Port Scan"
def test_title_case(self) -> None:
- from apps.mantis_web.data import fmt_attack
+ from apps.threat_model.data import fmt_attack
assert fmt_attack("brute_force_login") == "Brute Force Login"
def test_no_underscore(self) -> None:
- from apps.mantis_web.data import fmt_attack
+ from apps.threat_model.data import fmt_attack
assert fmt_attack("malware") == "Malware"
@@ -65,29 +65,29 @@ def test_no_underscore(self) -> None:
class TestCountryFlag:
def test_us_flag(self) -> None:
- from apps.mantis_web.data import country_flag
+ from apps.threat_model.data import country_flag
result = country_flag("US")
assert len(result) == 2 # two regional indicator symbols
def test_gb_flag(self) -> None:
- from apps.mantis_web.data import country_flag
+ from apps.threat_model.data import country_flag
result = country_flag("GB")
assert result != ""
def test_empty_string_returns_empty(self) -> None:
- from apps.mantis_web.data import country_flag
+ from apps.threat_model.data import country_flag
assert country_flag("") == ""
def test_non_two_char_returns_empty(self) -> None:
- from apps.mantis_web.data import country_flag
+ from apps.threat_model.data import country_flag
assert country_flag("USA") == ""
def test_lowercase_works(self) -> None:
- from apps.mantis_web.data import country_flag
+ from apps.threat_model.data import country_flag
assert country_flag("de") == country_flag("DE")
@@ -99,22 +99,22 @@ def test_lowercase_works(self) -> None:
class TestDaysBetween:
def test_same_date(self) -> None:
- from apps.mantis_web.data import days_between
+ from apps.threat_model.data import days_between
assert days_between("2024-01-01", "2024-01-01") == 0
def test_one_week(self) -> None:
- from apps.mantis_web.data import days_between
+ from apps.threat_model.data import days_between
assert days_between("2024-01-01", "2024-01-08") == 7
def test_invalid_date_returns_zero(self) -> None:
- from apps.mantis_web.data import days_between
+ from apps.threat_model.data import days_between
assert days_between("", "") == 0
def test_malformed_date_returns_zero(self) -> None:
- from apps.mantis_web.data import days_between
+ from apps.threat_model.data import days_between
assert days_between("not-a-date", "2024-01-01") == 0
@@ -126,7 +126,7 @@ def test_malformed_date_returns_zero(self) -> None:
class TestMaliciousRow:
def test_basic_fields(self) -> None:
- from apps.mantis_web.data import _malicious_row
+ from apps.threat_model.data import _malicious_row
raw = {
"ip": "198.51.100.1",
@@ -144,7 +144,7 @@ def test_basic_fields(self) -> None:
assert "spamhaus_drop" in row["blocklist_str"]
def test_missing_optional_fields_use_defaults(self) -> None:
- from apps.mantis_web.data import _malicious_row
+ from apps.threat_model.data import _malicious_row
raw = {"ip": "203.0.113.1"}
row = _malicious_row(raw)
@@ -154,7 +154,7 @@ def test_missing_optional_fields_use_defaults(self) -> None:
assert row["attack_types"] == []
def test_empty_blocklists_shows_dash(self) -> None:
- from apps.mantis_web.data import _malicious_row
+ from apps.threat_model.data import _malicious_row
raw = {"ip": "1.2.3.4", "blocklists": []}
row = _malicious_row(raw)
@@ -170,7 +170,7 @@ def test_empty_blocklists_shows_dash(self) -> None:
def _get_app_helpers():
"""Import _page_args and _sort_rows with data module mocked."""
for key in list(sys.modules):
- if "apps.mantis_web" in key:
+ if "apps.threat_model" in key:
del sys.modules[key]
mock_data = MagicMock()
@@ -192,8 +192,8 @@ def _get_app_helpers():
mock_data._fp_row = MagicMock(return_value={})
mock_data._malicious_row = MagicMock(return_value={})
- with patch.dict(sys.modules, {"apps.mantis_web.data": mock_data}):
- import apps.mantis_web.app as app_mod
+ with patch.dict(sys.modules, {"apps.threat_model.data": mock_data}):
+ import apps.threat_model.app as app_mod
return app_mod._page_args, app_mod._sort_rows
@@ -228,6 +228,60 @@ def test_invalid_per_page_defaults(self) -> None:
assert per_page == 50
+# ---------------------------------------------------------------------------
+# get_tickets_for_ip — must use TICKETS_BY_IP index, not scan _raw_tickets
+# ---------------------------------------------------------------------------
+
+
+class TestGetTicketsForIp:
+ def test_returns_ticket_for_known_ip(self) -> None:
+ import apps.threat_model.data as data_mod
+
+ ticket = {"id": "1", "created_at": "2024-01-01", "ips": ["1.2.3.4"]}
+ with (
+ patch.object(data_mod, "TICKETS_BY_IP", {"1.2.3.4": ["1"]}),
+ patch.object(data_mod, "TICKETS_BY_ID", {"1": ticket}),
+ ):
+ result = data_mod.get_tickets_for_ip("1.2.3.4")
+ assert result == [ticket]
+
+ def test_returns_empty_for_unknown_ip(self) -> None:
+ import apps.threat_model.data as data_mod
+
+ with (
+ patch.object(data_mod, "TICKETS_BY_IP", {}),
+ patch.object(data_mod, "TICKETS_BY_ID", {}),
+ ):
+ result = data_mod.get_tickets_for_ip("9.9.9.9")
+ assert result == []
+
+ def test_sorted_newest_first(self) -> None:
+ import apps.threat_model.data as data_mod
+
+ t1 = {"id": "1", "created_at": "2024-01-01"}
+ t2 = {"id": "2", "created_at": "2024-06-01"}
+ t3 = {"id": "3", "created_at": "2024-03-01"}
+ with (
+ patch.object(data_mod, "TICKETS_BY_IP", {"1.2.3.4": ["1", "2", "3"]}),
+ patch.object(data_mod, "TICKETS_BY_ID", {"1": t1, "2": t2, "3": t3}),
+ ):
+ result = data_mod.get_tickets_for_ip("1.2.3.4")
+ assert result[0]["id"] == "2"
+ assert result[-1]["id"] == "1"
+
+ def test_skips_missing_ticket_ids(self) -> None:
+ import apps.threat_model.data as data_mod
+
+ ticket = {"id": "1", "created_at": "2024-01-01"}
+ with (
+ patch.object(data_mod, "TICKETS_BY_IP", {"1.2.3.4": ["1", "999"]}),
+ patch.object(data_mod, "TICKETS_BY_ID", {"1": ticket}),
+ ):
+ result = data_mod.get_tickets_for_ip("1.2.3.4")
+ assert len(result) == 1
+ assert result[0]["id"] == "1"
+
+
class TestSortRows:
def setup_method(self) -> None:
_, self._sort_rows = _get_app_helpers()
diff --git a/tests/test_zeek_base.py b/tests/test_zeek_base.py
index 4a6e7f8..ada79b9 100644
--- a/tests/test_zeek_base.py
+++ b/tests/test_zeek_base.py
@@ -116,7 +116,7 @@ def test_build_base_query_timestamp_must() -> None:
sensors=None,
datasets=["conn"],
)
- must = body["query"]["bool"]["must"]
+ must = body["query"]["bool"]["filter"]
assert any("range" in c and "@timestamp" in c["range"] for c in must)
@@ -130,7 +130,7 @@ def test_build_base_query_dataset_filter() -> None:
sensors=None,
datasets=["dns"],
)
- must = body["query"]["bool"]["must"]
+ must = body["query"]["bool"]["filter"]
dataset_clause = next((c for c in must if "terms" in c and "event.dataset" in c["terms"]), None)
assert dataset_clause is not None
assert dataset_clause["terms"]["event.dataset"] == ["dns"]
@@ -146,7 +146,7 @@ def test_build_base_query_all_datasets_omits_filter() -> None:
sensors=None,
datasets=["all"],
)
- must = body["query"]["bool"]["must"]
+ must = body["query"]["bool"]["filter"]
assert not any("terms" in c and "event.dataset" in c.get("terms", {}) for c in must)
@@ -160,7 +160,7 @@ def test_build_base_query_sensor_filter() -> None:
sensors=["hedgehog-example"],
datasets=["conn"],
)
- must = body["query"]["bool"]["must"]
+ must = body["query"]["bool"]["filter"]
sensor_clause = next((c for c in must if "terms" in c and "host.name" in c["terms"]), None)
assert sensor_clause is not None
assert "hedgehog-example" in sensor_clause["terms"]["host.name"]
@@ -177,7 +177,7 @@ def test_build_base_query_src_ip_filter() -> None:
datasets=["conn"],
src_ip_filter="198.51.100.1",
)
- must = body["query"]["bool"]["must"]
+ must = body["query"]["bool"]["filter"]
ip_clause = next((c for c in must if "term" in c and "source.ip" in c["term"]), None)
assert ip_clause is not None
assert ip_clause["term"]["source.ip"] == "198.51.100.1"
diff --git a/uv.lock b/uv.lock
index 333a256..8c35a99 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1480,7 +1480,7 @@ wheels = [
[[package]]
name = "pisces-scripts"
-version = "0.1.0"
+version = "1.0.0"
source = { virtual = "." }
dependencies = [
{ name = "beautifulsoup4" },