From 1effa0a04931cd749400520c7af3ca08e91a5048 Mon Sep 17 00:00:00 2001 From: deep60 Date: Thu, 20 Aug 2026 18:16:10 +0530 Subject: [PATCH 1/3] fix(api): return the user's own submissions from GET /submissions `create_file_submission` writes to the `submissions` table (what a user uploads), but `list_submissions` delegated to `get_submissions`, which reads `bounty_submissions` (an analyst's verdict against a bounty). Two different tables, two different meanings. The result was that anything a user submitted was returned by no list endpoint at all: submit a file, get a 200 and an id, then watch it vanish from the dashboard and marketplace. Verified against the live deployment - after submitting, gateway.submissions held 1 row while GET /submissions returned {"submissions":[]} from bounty_submissions. Now returns the caller's own submissions, newest first, as a flat camelCase array. Both details match what the client already expects: marketplace.tsx declares useQuery, and the default fetcher in queryClient.ts does no unwrapping, so the previous envelope object arrived where an array was required. `status` and `analysisStatus` are both emitted because ApiSubmission reads each in different places. Scoped to the authenticated submitter - the route is in the strict-auth group and returning other users' uploads would leak filenames and descriptions. analysisType/bountyAmount/priority/description are lifted out of the row's metadata JSON, which is where create_file_submission puts them. --- .../api-gateway/src/handlers/submission.rs | 70 +++++++++++++++++-- backend/api-gateway/src/services/database.rs | 65 +++++++++++++++++ 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/backend/api-gateway/src/handlers/submission.rs b/backend/api-gateway/src/handlers/submission.rs index 2be0686ad..f6b61f7f2 100644 --- a/backend/api-gateway/src/handlers/submission.rs +++ b/backend/api-gateway/src/handlers/submission.rs @@ -63,6 +63,41 @@ pub struct FileSubmissionResponse { pub created_at: DateTime, } +/// One row of `GET /api/v1/submissions`. +/// +/// Serialized camelCase and returned as a FLAT ARRAY because that is what the +/// client already expects: `marketplace.tsx` declares +/// `useQuery({ queryKey: ["/api/submissions"] })` and the +/// default fetcher in `queryClient.ts` does no unwrapping, so an envelope +/// object would arrive where an array is required. +/// +/// `analysisType`, `bountyAmount` and `priority` live inside the row's +/// `metadata` JSON (that is how `create_file_submission` writes them), so they +/// are lifted back out here rather than exposing the raw blob. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FileSubmissionListItem { + pub id: Uuid, + pub submitter_id: Uuid, + pub original_filename: Option, + pub file_hash: Option, + pub url: Option, + pub file_size: Option, + pub mime_type: Option, + pub submission_type: String, + pub analysis_status: String, + /// Duplicate of `analysis_status`. `ApiSubmission` reads `status` in some + /// places and `analysisStatus` in others; emitting both keeps every + /// existing call site working without touching the client. + pub status: String, + pub is_malicious: Option, + pub description: Option, + pub analysis_type: Option, + pub bounty_amount: Option, + pub priority: Option, + pub created_at: DateTime, +} + #[derive(Deserialize)] pub struct UpdateSubmissionRequest { pub analysis_summary: Option, @@ -778,12 +813,37 @@ pub fn create_submission_router() -> Router { // Aliases / stubs for v1 routes -/// List submissions (alias for get_submissions) +/// `GET /api/v1/submissions` — the caller's own file/URL submissions. +/// +/// Previously this delegated to `get_submissions`, which reads +/// `bounty_submissions` (analyst verdicts against a bounty). But +/// `create_file_submission` writes to `submissions` (what a user uploads), so +/// anything a user submitted was never returned by any list endpoint — it +/// simply disappeared from the UI. These are two different tables with two +/// different meanings, and this route is the one the client uses for "my +/// submissions". +/// +/// Scoped to the authenticated submitter: the route sits in the protected +/// group, and returning other users' uploads here would leak filenames and +/// descriptions to anyone with an account. pub async fn list_submissions( - state: State, - query: Query, -) -> Result, StatusCode> { - get_submissions(state, query).await + State(state): State, + claims: crate::middleware::auth::Claims, + Query(filters): Query, +) -> Result>, StatusCode> { + let limit = filters.limit.unwrap_or(50).clamp(1, 200) as i64; + let offset = (filters.page.unwrap_or(1).max(1) as i64 - 1) * limit; + + let rows = state + .db + .list_file_submissions_for_user(claims.sub, limit, offset) + .await + .map_err(|e| { + tracing::error!("Failed to list submissions for {}: {}", claims.sub, e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(Json(rows)) } /// Get single submission (alias for get_submission_details) diff --git a/backend/api-gateway/src/services/database.rs b/backend/api-gateway/src/services/database.rs index 7e2472da3..091573624 100644 --- a/backend/api-gateway/src/services/database.rs +++ b/backend/api-gateway/src/services/database.rs @@ -507,6 +507,71 @@ impl DatabaseService { Ok(()) } + /// List one user's own file/URL submissions, newest first. + /// + /// Reads `submissions` (what a user uploaded), NOT `bounty_submissions` + /// (an analyst's verdict on a bounty) — see `list_submissions` in the + /// handler for why that distinction matters. + /// + /// `analysis_type` / `bounty_amount` / `priority` / `description` are + /// stored inside `metadata` by `create_file_submission`, so they are + /// extracted here with `->>` rather than being real columns. + pub async fn list_file_submissions_for_user( + &self, + submitter_id: Uuid, + limit: i64, + offset: i64, + ) -> Result> { + let rows = sqlx::query( + r#" + SELECT + id, submitter_id, original_filename, file_hash, url, + file_size, mime_type, submission_type, + COALESCE(analysis_status, 'pending') AS analysis_status, + is_malicious, created_at, + metadata ->> 'description' AS description, + metadata ->> 'analysisType' AS analysis_type, + metadata ->> 'bountyAmount' AS bounty_amount, + (metadata ->> 'priority')::boolean AS priority + FROM submissions + WHERE submitter_id = $1 + ORDER BY created_at DESC + LIMIT $2 OFFSET $3 + "#, + ) + .bind(submitter_id) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await + .context("Failed to list file submissions for user")?; + + Ok(rows + .into_iter() + .map(|r| { + let analysis_status: String = r.get("analysis_status"); + crate::handlers::submission::FileSubmissionListItem { + id: r.get("id"), + submitter_id: r.get("submitter_id"), + original_filename: r.get("original_filename"), + file_hash: r.get("file_hash"), + url: r.get("url"), + file_size: r.get("file_size"), + mime_type: r.get("mime_type"), + submission_type: r.get("submission_type"), + status: analysis_status.clone(), + analysis_status, + is_malicious: r.get("is_malicious"), + description: r.get("description"), + analysis_type: r.get("analysis_type"), + bounty_amount: r.get("bounty_amount"), + priority: r.get("priority"), + created_at: r.get("created_at"), + } + }) + .collect()) + } + /// Get submissions with filters (paginated) pub async fn get_submissions_with_filters( &self, From 6fced78ac3df47e36496069ed795504b08d230ed Mon Sep 17 00:00:00 2001 From: deep60 Date: Thu, 20 Aug 2026 23:06:38 +0530 Subject: [PATCH 2/3] fix(ci): clear the two clippy errors blocking Rust CI Rust CI has failed on main for the last four runs, so every PR inherits a red `build` check. Both errors are `-D warnings` promotions from a newer clippy and are unrelated to any recent change: api-gateway/src/handlers/websocket.rs:92 useless conversion to the same type: `String` (drop `.into()`) analysis-engine/src/analyzers/static_analyzer.rs:707 casting to the same type is unnecessary (`u32` -> `u32`) (drop the cast) Fixed here rather than in a separate PR because they block this one from merging; neither file is otherwise touched by this branch. --- backend/analysis-engine/src/analyzers/static_analyzer.rs | 2 +- backend/api-gateway/src/handlers/websocket.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/analysis-engine/src/analyzers/static_analyzer.rs b/backend/analysis-engine/src/analyzers/static_analyzer.rs index 4a6dfe7b3..35e7ff0d4 100644 --- a/backend/analysis-engine/src/analyzers/static_analyzer.rs +++ b/backend/analysis-engine/src/analyzers/static_analyzer.rs @@ -704,7 +704,7 @@ impl StaticAnalyzer { Ok(PEAnalysis { machine_type: format!("{:?}", pe.header.coff_header.machine), timestamp: Some(pe.header.coff_header.time_date_stamp), - entry_point: pe.entry as u32, + entry_point: pe.entry, sections, imports, exports, diff --git a/backend/api-gateway/src/handlers/websocket.rs b/backend/api-gateway/src/handlers/websocket.rs index 4f2358ede..3e97a9800 100644 --- a/backend/api-gateway/src/handlers/websocket.rs +++ b/backend/api-gateway/src/handlers/websocket.rs @@ -89,7 +89,7 @@ fn axum_to_tungstenite(msg: Message) -> tokio_tungstenite::tungstenite::Message // bridge via &str. Binary/Ping/Pong share bytes::Bytes so pass through. // axum 0.8 uses Bytes / its own Utf8Bytes; tungstenite 0.24 uses Vec // for Binary/Ping/Pong and its own Utf8Bytes for Text. Bridge explicitly. - Message::Text(t) => TM::Text(t.to_string().into()), + Message::Text(t) => TM::Text(t.to_string()), Message::Binary(b) => TM::Binary(b.to_vec()), Message::Ping(p) => TM::Ping(p.to_vec()), Message::Pong(p) => TM::Pong(p.to_vec()), From f33b72c354bcdcd4295376f1950a684837792d28 Mon Sep 17 00:00:00 2001 From: deep60 Date: Thu, 27 Aug 2026 16:26:39 +0530 Subject: [PATCH 3/3] fix(analysis-engine): stop truncating the ML feature vector --- backend/analysis-engine/ml_models/README.md | 18 +++- .../src/analyzers/ml_analyzer.rs | 87 ++++++++++++++++--- backend/analysis-engine/src/config.rs | 17 ++-- 3 files changed, 96 insertions(+), 26 deletions(-) diff --git a/backend/analysis-engine/ml_models/README.md b/backend/analysis-engine/ml_models/README.md index 45cb23dc9..2fd52906d 100644 --- a/backend/analysis-engine/ml_models/README.md +++ b/backend/analysis-engine/ml_models/README.md @@ -19,7 +19,7 @@ The analyzer feeds each model a single input tensor and reads a single output tensor (bound positionally, so input/output *names* don't matter): - **Input:** `float32`, shape `[1, feature_size]` (`feature_size` defaults to - 256, configurable via `ML_FEATURE_SIZE`). + **259**, configurable via `ML_FEATURE_SIZE`). - **Classifier output:** `float32`, shape `[1, n_classes]` — a per-class score vector. `argmax` selects the class; index 0 is treated as `benign`. The label list is index-aligned (see `MlAnalyzerConfig::labels`). @@ -32,10 +32,22 @@ tensor (bound positionally, so input/output *names* don't matter): ``` [ size_norm, entropy/8, printable_ratio, <256-bin byte-frequency histogram> ] + \_____________ 3 scalars ______________/ \________ 256 bins ________/ ``` -padded/truncated to `feature_size`. **A model must be trained against this exact -layout** (or update `extract_features` to match your training pipeline). +which is `3 + 256 = 259` floats — the default `feature_size`, so the layout is +passed through intact. **A model must be trained against this exact layout** (or +update `extract_features` to match your training pipeline). + +> Setting `ML_FEATURE_SIZE` *below* 259 truncates the tail of the histogram +> (the analyzer logs a warning at load time saying how many bins survive); +> setting it above 259 zero-pads. Only do either if your model was trained on +> the corresponding shape. +> +> Historical note: `feature_size` defaulted to 256 while the extractor emitted +> 259 values, so histogram bins 253-255 were dropped without warning. Models +> trained against that old truncated layout need `ML_FEATURE_SIZE=256` or a +> retrain. ## Enabling ML in a build diff --git a/backend/analysis-engine/src/analyzers/ml_analyzer.rs b/backend/analysis-engine/src/analyzers/ml_analyzer.rs index a7ef21055..61d98d4f4 100644 --- a/backend/analysis-engine/src/analyzers/ml_analyzer.rs +++ b/backend/analysis-engine/src/analyzers/ml_analyzer.rs @@ -14,8 +14,11 @@ //! The feature-vector layout produced by [`extract_features`] is a contract //! the trained model must match. The default layout is: //! [ size_norm, entropy_norm, printable_ratio, <256-bin byte histogram> ] -//! padded/truncated to `feature_size`. Adjust both sides together if you train -//! a model with a different input. +//! which is exactly [`DEFAULT_FEATURE_SIZE`] (= 259) floats. `feature_size` +//! defaults to that, so the natural layout survives intact; overriding it to a +//! smaller value truncates the tail of the histogram (and is warned about at +//! load time). Adjust both sides together if you train a model with a +//! different input. use std::path::Path; use std::sync::Mutex; @@ -30,6 +33,21 @@ use crate::models::analysis_result::{ DetectionResult, EngineType, SeverityLevel, ThreatCategory, ThreatVerdict, }; +/// Number of scalar features emitted before the byte histogram +/// (`size_norm`, `entropy_norm`, `printable_ratio`). +pub const SCALAR_FEATURES: usize = 3; + +/// Number of bins in the byte-frequency histogram (one per possible byte). +pub const HISTOGRAM_BINS: usize = 256; + +/// Natural length of the feature vector produced by +/// [`MlAnalyzer::extract_features`]: the scalars plus the full histogram. +/// +/// This is the default `feature_size`. A smaller `feature_size` silently drops +/// the tail of the histogram, so the default must not be lowered without +/// retraining the models against the shorter layout. +pub const DEFAULT_FEATURE_SIZE: usize = SCALAR_FEATURES + HISTOGRAM_BINS; + /// Configuration for the ML analyzer. #[derive(Debug, Clone)] pub struct MlAnalyzerConfig { @@ -59,7 +77,7 @@ impl Default for MlAnalyzerConfig { feature_size: std::env::var("ML_FEATURE_SIZE") .ok() .and_then(|s| s.parse().ok()) - .unwrap_or(256), + .unwrap_or(DEFAULT_FEATURE_SIZE), anomaly_threshold: std::env::var("ML_ANOMALY_THRESHOLD") .ok() .and_then(|s| s.parse().ok()) @@ -103,6 +121,17 @@ impl MlAnalyzer { }; } + if config.feature_size < DEFAULT_FEATURE_SIZE { + warn!( + "ML_FEATURE_SIZE={} is below the natural layout length of {}; the byte \ + histogram will be truncated to its first {} bins. Models must be trained \ + against this exact shortened layout.", + config.feature_size, + DEFAULT_FEATURE_SIZE, + config.feature_size.saturating_sub(SCALAR_FEATURES), + ); + } + let classifier = Self::try_load_session("threat classifier", &config.classifier_model_path); let anomaly = Self::try_load_session("anomaly detector", &config.anomaly_model_path); @@ -137,7 +166,7 @@ impl MlAnalyzer { } } - match Session::builder().and_then(|b| b.commit_from_file(p)) { + match Session::builder().and_then(|mut b| b.commit_from_file(p)) { Ok(session) => { info!("Loaded ML {label} model from {path}"); Some(Mutex::new(session)) @@ -341,9 +370,12 @@ impl MlAnalyzer { /// Extract a fixed-length numeric feature vector from raw bytes. /// - /// Layout: [size_norm, entropy_norm, printable_ratio, 256-bin histogram], - /// then padded/truncated to `feature_size`. This is a deterministic, - /// model-agnostic baseline; retrain-time feature engineering must match it. + /// Layout: [size_norm, entropy_norm, printable_ratio, 256-bin histogram] — + /// [`DEFAULT_FEATURE_SIZE`] floats — then padded or truncated to + /// `feature_size`. At the default `feature_size` the layout is preserved + /// exactly; a smaller configured size truncates the histogram tail (warned + /// about at load time). This is a deterministic, model-agnostic baseline; + /// retrain-time feature engineering must match it. fn extract_features(&self, data: &[u8]) -> Vec { let mut features = Vec::with_capacity(self.config.feature_size); @@ -375,14 +407,15 @@ impl MlAnalyzer { features.push(entropy / 8.0); // entropy is 0..8 bits features.push(printable_ratio); - // 256-bin normalized byte-frequency histogram. + // Normalized byte-frequency histogram, one bin per byte value. if !data.is_empty() { for c in counts.iter() { features.push(*c as f32 / len); } } else { - features.extend(std::iter::repeat(0.0).take(256)); + features.extend(std::iter::repeat(0.0).take(HISTOGRAM_BINS)); } + debug_assert_eq!(features.len(), DEFAULT_FEATURE_SIZE); features.resize(self.config.feature_size, 0.0); features @@ -426,6 +459,36 @@ mod tests { assert_eq!(empty.len(), analyzer.config.feature_size); } + /// Regression: the default `feature_size` used to be 256 while the layout + /// emits 259 floats, so `resize` silently dropped histogram bins 253-255. + /// The default must keep every bin. + #[test] + fn test_default_feature_size_preserves_full_histogram() { + assert_eq!(DEFAULT_FEATURE_SIZE, SCALAR_FEATURES + HISTOGRAM_BINS); + assert_eq!(MlAnalyzerConfig::default().feature_size, DEFAULT_FEATURE_SIZE); + + let analyzer = MlAnalyzer::new(disabled_config()); + + // A sample containing only byte 0xFF must light up the *last* bin, + // which truncation to 256 would have removed. + let feats = analyzer.extract_features(&[0xFFu8; 4]); + assert_eq!(feats.len(), DEFAULT_FEATURE_SIZE); + assert_eq!(feats[SCALAR_FEATURES + 0xFF], 1.0); + assert_eq!(feats[SCALAR_FEATURES + 0x00], 0.0); + } + + /// An explicit undersized override still truncates, deliberately. + #[test] + fn test_explicit_smaller_feature_size_truncates() { + let cfg = MlAnalyzerConfig { + enabled: false, + feature_size: 64, + ..Default::default() + }; + let analyzer = MlAnalyzer::new(cfg); + assert_eq!(analyzer.extract_features(b"hello world").len(), 64); + } + #[tokio::test] async fn test_disabled_returns_unknown() { let analyzer = MlAnalyzer::new(disabled_config()); @@ -452,8 +515,8 @@ mod tests { /// End-to-end inference against a real ONNX model. Self-skips unless /// `ML_TEST_CLASSIFIER` points at a model whose single input is - /// `[1, feature_size]` f32 and single output is the class vector. The - /// fixture used in local verification drives class index 1 ("malware"). + /// `[1, DEFAULT_FEATURE_SIZE]` f32 and single output is the class vector. + /// The fixture used in local verification drives class index 1 ("malware"). #[tokio::test] async fn test_real_model_inference() { let Ok(path) = std::env::var("ML_TEST_CLASSIFIER") else { @@ -464,7 +527,7 @@ mod tests { enabled: true, classifier_model_path: path, anomaly_model_path: "/nonexistent/anomaly.onnx".to_string(), - feature_size: 256, + feature_size: DEFAULT_FEATURE_SIZE, ..Default::default() }; let analyzer = MlAnalyzer::new(cfg); diff --git a/backend/analysis-engine/src/config.rs b/backend/analysis-engine/src/config.rs index 817683094..083f8c9c7 100644 --- a/backend/analysis-engine/src/config.rs +++ b/backend/analysis-engine/src/config.rs @@ -535,15 +535,19 @@ impl Default for SandboxConfig { } /// Analyzers configuration +/// +/// NOTE: the ML analyzer is **not** configured here. It owns its own settings +/// in [`crate::analyzers::MlAnalyzerConfig`], read from `ENABLE_ML_ENGINE`, +/// `ML_CLASSIFIER_MODEL`, `ML_ANOMALY_MODEL`, `ML_FEATURE_SIZE` and +/// `ML_ANOMALY_THRESHOLD`. Duplicate `ENABLE_ML_ANALYZER` / `ML_MODEL_PATH` +/// fields used to live here and were read by nothing. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AnalyzersConfig { pub enable_static_analyzer: bool, pub enable_dynamic_analyzer: bool, pub enable_hash_analyzer: bool, pub enable_yara_engine: bool, - pub enable_ml_analyzer: bool, pub yara_rules_directory: PathBuf, - pub ml_model_path: PathBuf, pub analysis_timeout_seconds: u64, pub max_concurrent_analyses: usize, pub enable_parallel_analysis: bool, @@ -568,16 +572,9 @@ impl AnalyzersConfig { .unwrap_or_else(|_| "true".to_string()) .parse() .unwrap_or(true), - enable_ml_analyzer: env::var("ENABLE_ML_ANALYZER") - .unwrap_or_else(|_| "false".to_string()) - .parse() - .unwrap_or(false), yara_rules_directory: PathBuf::from( env::var("YARA_RULES_DIR").unwrap_or_else(|_| "./rules".to_string()), ), - ml_model_path: PathBuf::from( - env::var("ML_MODEL_PATH").unwrap_or_else(|_| "./models/malware_detector.onnx".to_string()), - ), analysis_timeout_seconds: env::var("ANALYSIS_TIMEOUT") .unwrap_or_else(|_| "300".to_string()) .parse() @@ -611,9 +608,7 @@ impl Default for AnalyzersConfig { enable_dynamic_analyzer: true, enable_hash_analyzer: true, enable_yara_engine: true, - enable_ml_analyzer: false, yara_rules_directory: PathBuf::from("./rules"), - ml_model_path: PathBuf::from("./models/malware_detector.onnx"), analysis_timeout_seconds: 300, max_concurrent_analyses: 10, enable_parallel_analysis: true,