fix/ml feature vector truncation

backend/analysis-engine/ml_models/README.md

@@ -19,7 +19,7 @@ The analyzer feeds each model a single input tensor and reads a single output
 tensor (bound positionally, so input/output *names* don't matter):
 
 - **Input:** `float32`, shape `[1, feature_size]` (`feature_size` defaults to
-  256, configurable via `ML_FEATURE_SIZE`).
+  **259**, configurable via `ML_FEATURE_SIZE`).
 - **Classifier output:** `float32`, shape `[1, n_classes]` — a per-class score
   vector. `argmax` selects the class; index 0 is treated as `benign`. The label
   list is index-aligned (see `MlAnalyzerConfig::labels`).
@@ -32,10 +32,22 @@ tensor (bound positionally, so input/output *names* don't matter):
 
 ```
 [ size_norm, entropy/8, printable_ratio, <256-bin byte-frequency histogram> ]
+   \_____________ 3 scalars ______________/  \________ 256 bins ________/
 ```
 
-padded/truncated to `feature_size`. **A model must be trained against this exact
-layout** (or update `extract_features` to match your training pipeline).
+which is `3 + 256 = 259` floats — the default `feature_size`, so the layout is
+passed through intact. **A model must be trained against this exact layout** (or
+update `extract_features` to match your training pipeline).
+
+> Setting `ML_FEATURE_SIZE` *below* 259 truncates the tail of the histogram
+> (the analyzer logs a warning at load time saying how many bins survive);
+> setting it above 259 zero-pads. Only do either if your model was trained on
+> the corresponding shape.
+>
+> Historical note: `feature_size` defaulted to 256 while the extractor emitted
+> 259 values, so histogram bins 253-255 were dropped without warning. Models
+> trained against that old truncated layout need `ML_FEATURE_SIZE=256` or a
+> retrain.
 
 ## Enabling ML in a build
 

backend/analysis-engine/src/analyzers/ml_analyzer.rs

@@ -14,8 +14,11 @@
 //! The feature-vector layout produced by [`extract_features`] is a contract
 //! the trained model must match. The default layout is:
 //!   [ size_norm, entropy_norm, printable_ratio, <256-bin byte histogram> ]
-//! padded/truncated to `feature_size`. Adjust both sides together if you train
-//! a model with a different input.
+//! which is exactly [`DEFAULT_FEATURE_SIZE`] (= 259) floats. `feature_size`
+//! defaults to that, so the natural layout survives intact; overriding it to a
+//! smaller value truncates the tail of the histogram (and is warned about at
+//! load time). Adjust both sides together if you train a model with a
+//! different input.
 
 use std::path::Path;
 use std::sync::Mutex;
@@ -30,6 +33,21 @@ use crate::models::analysis_result::{
     DetectionResult, EngineType, SeverityLevel, ThreatCategory, ThreatVerdict,
 };
 
+/// Number of scalar features emitted before the byte histogram
+/// (`size_norm`, `entropy_norm`, `printable_ratio`).
+pub const SCALAR_FEATURES: usize = 3;
+
+/// Number of bins in the byte-frequency histogram (one per possible byte).
+pub const HISTOGRAM_BINS: usize = 256;
+
+/// Natural length of the feature vector produced by
+/// [`MlAnalyzer::extract_features`]: the scalars plus the full histogram.
+///
+/// This is the default `feature_size`. A smaller `feature_size` silently drops
+/// the tail of the histogram, so the default must not be lowered without
+/// retraining the models against the shorter layout.
+pub const DEFAULT_FEATURE_SIZE: usize = SCALAR_FEATURES + HISTOGRAM_BINS;
+
 /// Configuration for the ML analyzer.
 #[derive(Debug, Clone)]
 pub struct MlAnalyzerConfig {
@@ -59,7 +77,7 @@ impl Default for MlAnalyzerConfig {
             feature_size: std::env::var("ML_FEATURE_SIZE")
                 .ok()
                 .and_then(|s| s.parse().ok())
-                .unwrap_or(256),
+                .unwrap_or(DEFAULT_FEATURE_SIZE),
             anomaly_threshold: std::env::var("ML_ANOMALY_THRESHOLD")
                 .ok()
                 .and_then(|s| s.parse().ok())
@@ -103,6 +121,17 @@ impl MlAnalyzer {
             };
         }
 
+        if config.feature_size < DEFAULT_FEATURE_SIZE {
+            warn!(
+                "ML_FEATURE_SIZE={} is below the natural layout length of {}; the byte \
+                 histogram will be truncated to its first {} bins. Models must be trained \
+                 against this exact shortened layout.",
+                config.feature_size,
+                DEFAULT_FEATURE_SIZE,
+                config.feature_size.saturating_sub(SCALAR_FEATURES),
+            );
+        }
+
         let classifier = Self::try_load_session("threat classifier", &config.classifier_model_path);
         let anomaly = Self::try_load_session("anomaly detector", &config.anomaly_model_path);
 
@@ -137,7 +166,7 @@ impl MlAnalyzer {
             }
         }
 
-        match Session::builder().and_then(|b| b.commit_from_file(p)) {
+        match Session::builder().and_then(|mut b| b.commit_from_file(p)) {
             Ok(session) => {
                 info!("Loaded ML {label} model from {path}");
                 Some(Mutex::new(session))
@@ -341,9 +370,12 @@ impl MlAnalyzer {
 
     /// Extract a fixed-length numeric feature vector from raw bytes.
     ///
-    /// Layout: [size_norm, entropy_norm, printable_ratio, 256-bin histogram],
-    /// then padded/truncated to `feature_size`. This is a deterministic,
-    /// model-agnostic baseline; retrain-time feature engineering must match it.
+    /// Layout: [size_norm, entropy_norm, printable_ratio, 256-bin histogram] —
+    /// [`DEFAULT_FEATURE_SIZE`] floats — then padded or truncated to
+    /// `feature_size`. At the default `feature_size` the layout is preserved
+    /// exactly; a smaller configured size truncates the histogram tail (warned
+    /// about at load time). This is a deterministic, model-agnostic baseline;
+    /// retrain-time feature engineering must match it.
     fn extract_features(&self, data: &[u8]) -> Vec<f32> {
         let mut features = Vec::with_capacity(self.config.feature_size);
 
@@ -375,14 +407,15 @@ impl MlAnalyzer {
         features.push(entropy / 8.0); // entropy is 0..8 bits
         features.push(printable_ratio);
 
-        // 256-bin normalized byte-frequency histogram.
+        // Normalized byte-frequency histogram, one bin per byte value.
         if !data.is_empty() {
             for c in counts.iter() {
                 features.push(*c as f32 / len);
             }
         } else {
-            features.extend(std::iter::repeat(0.0).take(256));
+            features.extend(std::iter::repeat(0.0).take(HISTOGRAM_BINS));
         }
+        debug_assert_eq!(features.len(), DEFAULT_FEATURE_SIZE);
 
         features.resize(self.config.feature_size, 0.0);
         features
@@ -426,6 +459,36 @@ mod tests {
         assert_eq!(empty.len(), analyzer.config.feature_size);
     }
 
+    /// Regression: the default `feature_size` used to be 256 while the layout
+    /// emits 259 floats, so `resize` silently dropped histogram bins 253-255.
+    /// The default must keep every bin.
+    #[test]
+    fn test_default_feature_size_preserves_full_histogram() {
+        assert_eq!(DEFAULT_FEATURE_SIZE, SCALAR_FEATURES + HISTOGRAM_BINS);
+        assert_eq!(MlAnalyzerConfig::default().feature_size, DEFAULT_FEATURE_SIZE);
+
+        let analyzer = MlAnalyzer::new(disabled_config());
+
+        // A sample containing only byte 0xFF must light up the *last* bin,
+        // which truncation to 256 would have removed.
+        let feats = analyzer.extract_features(&[0xFFu8; 4]);
+        assert_eq!(feats.len(), DEFAULT_FEATURE_SIZE);
+        assert_eq!(feats[SCALAR_FEATURES + 0xFF], 1.0);
+        assert_eq!(feats[SCALAR_FEATURES + 0x00], 0.0);
+    }
+
+    /// An explicit undersized override still truncates, deliberately.
+    #[test]
+    fn test_explicit_smaller_feature_size_truncates() {
+        let cfg = MlAnalyzerConfig {
+            enabled: false,
+            feature_size: 64,
+            ..Default::default()
+        };
+        let analyzer = MlAnalyzer::new(cfg);
+        assert_eq!(analyzer.extract_features(b"hello world").len(), 64);
+    }
+
     #[tokio::test]
     async fn test_disabled_returns_unknown() {
         let analyzer = MlAnalyzer::new(disabled_config());
@@ -452,8 +515,8 @@ mod tests {
 
     /// End-to-end inference against a real ONNX model. Self-skips unless
     /// `ML_TEST_CLASSIFIER` points at a model whose single input is
-    /// `[1, feature_size]` f32 and single output is the class vector. The
-    /// fixture used in local verification drives class index 1 ("malware").
+    /// `[1, DEFAULT_FEATURE_SIZE]` f32 and single output is the class vector.
+    /// The fixture used in local verification drives class index 1 ("malware").
     #[tokio::test]
     async fn test_real_model_inference() {
         let Ok(path) = std::env::var("ML_TEST_CLASSIFIER") else {
@@ -464,7 +527,7 @@ mod tests {
             enabled: true,
             classifier_model_path: path,
             anomaly_model_path: "/nonexistent/anomaly.onnx".to_string(),
-            feature_size: 256,
+            feature_size: DEFAULT_FEATURE_SIZE,
             ..Default::default()
         };
         let analyzer = MlAnalyzer::new(cfg);

backend/analysis-engine/src/analyzers/static_analyzer.rs

@@ -704,7 +704,7 @@ impl StaticAnalyzer {
         Ok(PEAnalysis {
             machine_type: format!("{:?}", pe.header.coff_header.machine),
             timestamp: Some(pe.header.coff_header.time_date_stamp),
-            entry_point: pe.entry as u32,
+            entry_point: pe.entry,
             sections,
             imports,
             exports,

backend/analysis-engine/src/config.rs

@@ -535,15 +535,19 @@ impl Default for SandboxConfig {
 }
 
 /// Analyzers configuration
+///
+/// NOTE: the ML analyzer is **not** configured here. It owns its own settings
+/// in [`crate::analyzers::MlAnalyzerConfig`], read from `ENABLE_ML_ENGINE`,
+/// `ML_CLASSIFIER_MODEL`, `ML_ANOMALY_MODEL`, `ML_FEATURE_SIZE` and
+/// `ML_ANOMALY_THRESHOLD`. Duplicate `ENABLE_ML_ANALYZER` / `ML_MODEL_PATH`
+/// fields used to live here and were read by nothing.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AnalyzersConfig {
     pub enable_static_analyzer: bool,
     pub enable_dynamic_analyzer: bool,
     pub enable_hash_analyzer: bool,
     pub enable_yara_engine: bool,
-    pub enable_ml_analyzer: bool,
     pub yara_rules_directory: PathBuf,
-    pub ml_model_path: PathBuf,
     pub analysis_timeout_seconds: u64,
     pub max_concurrent_analyses: usize,
     pub enable_parallel_analysis: bool,
@@ -568,16 +572,9 @@ impl AnalyzersConfig {
                 .unwrap_or_else(|_| "true".to_string())
                 .parse()
                 .unwrap_or(true),
-            enable_ml_analyzer: env::var("ENABLE_ML_ANALYZER")
-                .unwrap_or_else(|_| "false".to_string())
-                .parse()
-                .unwrap_or(false),
             yara_rules_directory: PathBuf::from(
                 env::var("YARA_RULES_DIR").unwrap_or_else(|_| "./rules".to_string()),
             ),
-            ml_model_path: PathBuf::from(
-                env::var("ML_MODEL_PATH").unwrap_or_else(|_| "./models/malware_detector.onnx".to_string()),
-            ),
             analysis_timeout_seconds: env::var("ANALYSIS_TIMEOUT")
                 .unwrap_or_else(|_| "300".to_string())
                 .parse()
@@ -611,9 +608,7 @@ impl Default for AnalyzersConfig {
             enable_dynamic_analyzer: true,
             enable_hash_analyzer: true,
             enable_yara_engine: true,
-            enable_ml_analyzer: false,
             yara_rules_directory: PathBuf::from("./rules"),
-            ml_model_path: PathBuf::from("./models/malware_detector.onnx"),
             analysis_timeout_seconds: 300,
             max_concurrent_analyses: 10,
             enable_parallel_analysis: true,

backend/api-gateway/src/handlers/submission.rs

@@ -63,6 +63,41 @@ pub struct FileSubmissionResponse {
     pub created_at: DateTime<Utc>,
 }
 
+/// One row of `GET /api/v1/submissions`.
+///
+/// Serialized camelCase and returned as a FLAT ARRAY because that is what the
+/// client already expects: `marketplace.tsx` declares
+/// `useQuery<ApiSubmission[]>({ queryKey: ["/api/submissions"] })` and the
+/// default fetcher in `queryClient.ts` does no unwrapping, so an envelope
+/// object would arrive where an array is required.
+///
+/// `analysisType`, `bountyAmount` and `priority` live inside the row's
+/// `metadata` JSON (that is how `create_file_submission` writes them), so they
+/// are lifted back out here rather than exposing the raw blob.
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FileSubmissionListItem {
+    pub id: Uuid,
+    pub submitter_id: Uuid,
+    pub original_filename: Option<String>,
+    pub file_hash: Option<String>,
+    pub url: Option<String>,
+    pub file_size: Option<i64>,
+    pub mime_type: Option<String>,
+    pub submission_type: String,
+    pub analysis_status: String,
+    /// Duplicate of `analysis_status`. `ApiSubmission` reads `status` in some
+    /// places and `analysisStatus` in others; emitting both keeps every
+    /// existing call site working without touching the client.
+    pub status: String,
+    pub is_malicious: Option<bool>,
+    pub description: Option<String>,
+    pub analysis_type: Option<String>,
+    pub bounty_amount: Option<String>,
+    pub priority: Option<bool>,
+    pub created_at: DateTime<Utc>,
+}
+
 #[derive(Deserialize)]
 pub struct UpdateSubmissionRequest {
     pub analysis_summary: Option<String>,
@@ -778,12 +813,37 @@ pub fn create_submission_router() -> Router<AppState> {
 
 // Aliases / stubs for v1 routes
 
-/// List submissions (alias for get_submissions)
+/// `GET /api/v1/submissions` — the caller's own file/URL submissions.
+///
+/// Previously this delegated to `get_submissions`, which reads
+/// `bounty_submissions` (analyst verdicts against a bounty). But
+/// `create_file_submission` writes to `submissions` (what a user uploads), so
+/// anything a user submitted was never returned by any list endpoint — it
+/// simply disappeared from the UI. These are two different tables with two
+/// different meanings, and this route is the one the client uses for "my
+/// submissions".
+///
+/// Scoped to the authenticated submitter: the route sits in the protected
+/// group, and returning other users' uploads here would leak filenames and
+/// descriptions to anyone with an account.
 pub async fn list_submissions(
-    state: State<AppState>,
-    query: Query<SubmissionFilters>,
-) -> Result<Json<SubmissionListResponse>, StatusCode> {
-    get_submissions(state, query).await
+    State(state): State<AppState>,
+    claims: crate::middleware::auth::Claims,
+    Query(filters): Query<SubmissionFilters>,
+) -> Result<Json<Vec<FileSubmissionListItem>>, StatusCode> {
+    let limit = filters.limit.unwrap_or(50).clamp(1, 200) as i64;
+    let offset = (filters.page.unwrap_or(1).max(1) as i64 - 1) * limit;
+
+    let rows = state
+        .db
+        .list_file_submissions_for_user(claims.sub, limit, offset)
+        .await
+        .map_err(|e| {
+            tracing::error!("Failed to list submissions for {}: {}", claims.sub, e);
+            StatusCode::INTERNAL_SERVER_ERROR
+        })?;
+
+    Ok(Json(rows))
 }
 
 /// Get single submission (alias for get_submission_details)

backend/api-gateway/src/handlers/websocket.rs

@@ -89,7 +89,7 @@ fn axum_to_tungstenite(msg: Message) -> tokio_tungstenite::tungstenite::Message
         // bridge via &str. Binary/Ping/Pong share bytes::Bytes so pass through.
         // axum 0.8 uses Bytes / its own Utf8Bytes; tungstenite 0.24 uses Vec<u8>
         // for Binary/Ping/Pong and its own Utf8Bytes for Text. Bridge explicitly.
-        Message::Text(t) => TM::Text(t.to_string().into()),
+        Message::Text(t) => TM::Text(t.to_string()),
         Message::Binary(b) => TM::Binary(b.to_vec()),
         Message::Ping(p) => TM::Ping(p.to_vec()),
         Message::Pong(p) => TM::Pong(p.to_vec()),