improve outlier detection defaults

Author: shrimbly

README.md

@@ -65,10 +65,10 @@ Divides content into batches and selects the sharpest frame from each batch.
 --selection-method batched --batch-size 5 --batch-buffer 2
 ```
 
-### Outlier Removal
+### Outlier Detection
 Removes unusually blurry frames by comparing each frame to its neighbors.
 ```bash
---selection-method outlier-removal --outlier-window-size 15 --outlier-sensitivity 50
+--selection-method outlier-removal --outlier-window-size 15 --outlier-sensitivity 60
 ```
 
 ## Command Line Options
@@ -84,8 +84,8 @@ Removes unusually blurry frames by comparing each frame to its neighbors.
 - `--min-buffer <int>`: Minimum number of intervening frames between selected frames (best-n, default: 3)
 - `--batch-size <int>`: Frames per batch (batched, default: 5)
 - `--batch-buffer <int>`: Frames to skip between batches (batched, default: 2)
-- `--outlier-window-size <int>`: Neighbor comparison window (outlier-removal, default: 15)
-- `--outlier-sensitivity <int>`: Removal aggressiveness 0-100 (outlier-removal, default: 50)
+- `--outlier-window-size <int>`: Local comparison window, minimum 5 (outlier-removal, default: 15)
+- `--outlier-sensitivity <int>`: Detection sensitivity 0-100 (outlier-removal, default: 60)
 
 ## Examples
 

sharp_frames/processing/frame_saver.py

@@ -382,7 +382,7 @@ def _get_method_params_for_metadata(self, config: Dict[str, Any]) -> Dict[str, A
             })
         elif selection_method == 'outlier_removal':
             params.update({
-                "outlier_sensitivity": config.get('outlier_sensitivity', 50),
+                "outlier_sensitivity": config.get('outlier_sensitivity', 60),
                 "outlier_window_size": config.get('outlier_window_size', 15)
             })
 

sharp_frames/processing/frame_selector.py

@@ -4,6 +4,8 @@
 
 from ..models.frame_data import FrameData
 from ..selection_methods import (
+    OUTLIER_DEFAULT_SENSITIVITY,
+    OUTLIER_DEFAULT_WINDOW_SIZE,
     select_batched_frames_core,
     select_best_n_frames_core,
     select_outlier_removal_frames_core,
@@ -47,8 +49,8 @@ def select_frames(
         if method in {"outlier_removal", "outlier-removal"}:
             return self._select_outlier_removal_frames(
                 frames,
-                params.get("outlier_sensitivity", 50),
-                params.get("outlier_window_size", 15),
+                params.get("outlier_sensitivity", OUTLIER_DEFAULT_SENSITIVITY),
+                params.get("outlier_window_size", OUTLIER_DEFAULT_WINDOW_SIZE),
             )
         raise ValueError(f"Unsupported selection method: {method}")
 

sharp_frames/processing/tui_processor.py

@@ -339,13 +339,13 @@ def validate_selection_parameters(self, method: str, **params) -> tuple[bool, st
                 return False, "Parameter 'batch_buffer' must be a non-negative integer"
 
         elif method == 'outlier_removal':
-            outlier_sensitivity = params.get('outlier_sensitivity', 50)
+            outlier_sensitivity = params.get('outlier_sensitivity', 60)
             if not isinstance(outlier_sensitivity, int) or not (0 <= outlier_sensitivity <= 100):
                 return False, "Parameter 'outlier_sensitivity' must be an integer between 0 and 100"
 
             outlier_window_size = params.get('outlier_window_size', 15)
-            if not isinstance(outlier_window_size, int) or outlier_window_size <= 0:
-                return False, "Parameter 'outlier_window_size' must be a positive integer"
+            if not isinstance(outlier_window_size, int) or outlier_window_size < 5:
+                return False, "Parameter 'outlier_window_size' must be at least 5"
 
         else:
             return False, f"Unknown selection method: {method}"

sharp_frames/selection_methods.py

@@ -1,6 +1,7 @@
 """Canonical frame-selection algorithms shared by the CLI and TUI."""
 
 from collections import OrderedDict
+from math import log1p
 from statistics import median
 from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
 
@@ -10,6 +11,13 @@
 Frame = Dict[str, Any]
 PositionedFrame = Tuple[int, Frame]
 
+OUTLIER_DEFAULT_WINDOW_SIZE = 15
+OUTLIER_DEFAULT_SENSITIVITY = 60
+OUTLIER_MIN_WINDOW_SIZE = 5
+OUTLIER_MIN_RELATIVE_DROP = 0.05
+OUTLIER_MAX_RELATIVE_DROP = 0.30
+OUTLIER_MIN_ROBUST_THRESHOLD = 1.5
+
 
 def _source_key(frame: Frame) -> Optional[str]:
     """Return a stable source identifier, if the frame belongs to a video."""
@@ -191,35 +199,61 @@ def _outlier_positions(
     if sensitivity <= 0 or not group:
         return set()
 
-    scores = [float(frame.get("sharpnessScore", 0) or 0) for _, frame in group]
+    raw_scores = [
+        max(0.0, float(frame.get("sharpnessScore", 0) or 0))
+        for _, frame in group
+    ]
+    scores = [log1p(score) for score in raw_scores]
 
-    actual_window_size = max(1, window_size)
+    actual_window_size = max(OUTLIER_MIN_WINDOW_SIZE, window_size)
     if actual_window_size % 2 == 0:
         actual_window_size += 1
     half_window = actual_window_size // 2
     maximum_threshold = threshold_divisor if threshold_divisor > 0 else 4.0
     sensitivity_ratio = min(100, sensitivity) / 100
-    robust_threshold = 0.5 + ((maximum_threshold - 0.5) * (1 - sensitivity_ratio))
+    robust_threshold = OUTLIER_MIN_ROBUST_THRESHOLD + (
+        (maximum_threshold - OUTLIER_MIN_ROBUST_THRESHOLD)
+        * (1 - sensitivity_ratio)
+    )
+    minimum_relative_drop = OUTLIER_MAX_RELATIVE_DROP - (
+        (OUTLIER_MAX_RELATIVE_DROP - OUTLIER_MIN_RELATIVE_DROP)
+        * sensitivity_ratio
+    )
     outliers: Set[int] = set()
 
     for position, current_score in enumerate(scores):
         window_start = max(0, position - half_window)
         window_end = min(len(group), position + half_window + 1)
         neighbor_scores = scores[window_start:position] + scores[position + 1 : window_end]
-        if not neighbor_scores or len(neighbor_scores) < max(0, min_neighbors):
+        required_neighbors = max(2, min_neighbors)
+        if len(neighbor_scores) < required_neighbors:
             continue
         neighbor_median = median(neighbor_scores)
+        raw_neighbor_scores = (
+            raw_scores[window_start:position]
+            + raw_scores[position + 1 : window_end]
+        )
+        raw_neighbor_median = median(raw_neighbor_scores)
         absolute_deviations = [
             abs(score - neighbor_median) for score in neighbor_scores
         ]
         median_absolute_deviation = median(absolute_deviations)
         robust_scale = median_absolute_deviation * 1.4826
         if robust_scale == 0:
-            robust_scale = max(abs(neighbor_median) * 0.05, 1e-9)
+            robust_scale = max(abs(neighbor_median) * 0.01, 1e-9)
 
         deficit = neighbor_median - current_score
         robust_deficit = deficit / robust_scale
-        if deficit > 0 and robust_deficit > robust_threshold:
+        relative_drop = (
+            (raw_neighbor_median - raw_scores[position]) / raw_neighbor_median
+            if raw_neighbor_median > 0
+            else 0.0
+        )
+        if (
+            deficit > 0
+            and relative_drop >= minimum_relative_drop
+            and robust_deficit > robust_threshold
+        ):
             outliers.add(position)
     return outliers
 
@@ -284,7 +318,7 @@ def select_outlier_removal_frames(
         progress_bar.update(len(result))
     selected_count = sum(frame["selected"] for frame in result)
     print(
-        f"Outlier removal: Marked {len(result) - selected_count} outliers. "
+        f"Outlier detection: Marked {len(result) - selected_count} outliers. "
         f"Keeping {selected_count} frames."
     )
     return result

sharp_frames/selection_preview.py

@@ -4,6 +4,8 @@
 from typing import Any, Dict, List, Sequence, Tuple
 
 from .selection_methods import (
+    OUTLIER_DEFAULT_SENSITIVITY,
+    OUTLIER_DEFAULT_WINDOW_SIZE,
     select_batched_frames_core,
     select_best_n_frames_core,
     select_outlier_removal_frames_core,
@@ -33,8 +35,8 @@ def _select_for_preview(
     if method in {"outlier-removal", "outlier_removal"}:
         assessed = select_outlier_removal_frames_core(
             frames,
-            params.get("outlier_window_size", 15),
-            params.get("outlier_sensitivity", 50),
+            params.get("outlier_window_size", OUTLIER_DEFAULT_WINDOW_SIZE),
+            params.get("outlier_sensitivity", OUTLIER_DEFAULT_SENSITIVITY),
             params.get("min_neighbors", 3),
             params.get("threshold_divisor", 4.0),
         )

sharp_frames/sharp_frames.py

@@ -169,9 +169,9 @@ def main():
     parser.add_argument("--batch-buffer", type=int, default=2,
                        help="Number of frames to skip between batches (default: 2)")
     parser.add_argument("--outlier-window-size", type=int, default=15,
-                       help="Number of neighboring frames to compare for outlier detection (default: 15)")
-    parser.add_argument("--outlier-sensitivity", type=int, default=50,
-                       help="Sensitivity of outlier detection, 0-100 (default: 50)")
+                       help="Local comparison window for outlier detection, minimum 5 (default: 15)")
+    parser.add_argument("--outlier-sensitivity", type=int, default=60,
+                       help="Sensitivity of outlier detection, 0-100 (default: 60)")
     parser.add_argument("--width", type=int, default=0,
                        help="Width to resize output images (height will be adjusted proportionally, 0 for no resizing)")
     parser.add_argument("--interactive", action="store_true", help="Run in legacy terminal prompt mode")
@@ -323,7 +323,7 @@ def run_interactive_mode():
     batch_size = 5
     batch_buffer = 2
     outlier_window_size = 15
-    outlier_sensitivity = 50
+    outlier_sensitivity = 60
 
     # Get method-specific parameters
     if selection_method == "best-n":
@@ -333,8 +333,8 @@ def run_interactive_mode():
         batch_size = get_valid_int("Enter batch size", min_value=1, default=5)
         batch_buffer = get_valid_int("Enter batch buffer (frames/images to skip between batches)", min_value=0, default=2)
     elif selection_method == "outlier-removal":
-        outlier_window_size = get_valid_int("Enter window size for comparison", min_value=3, max_value=30, default=15)
-        outlier_sensitivity = get_valid_int("Enter sensitivity (0-100, higher is more aggressive)", min_value=0, max_value=100, default=50)
+        outlier_window_size = get_valid_int("Enter window size for comparison", min_value=5, max_value=31, default=15)
+        outlier_sensitivity = get_valid_int("Enter sensitivity (0-100, higher is more aggressive)", min_value=0, max_value=100, default=60)
 
     output_format = get_choice(
         "Choose output format for saved images (or first 3 letters)",
@@ -400,4 +400,4 @@ def run_interactive_mode():
     return 0 if success else 1
 
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main())

sharp_frames/sharp_frames_processor.py

@@ -56,7 +56,7 @@ def __init__(self,
 
                  # --- Parameters for 'outlier-removal' selection ---
                  outlier_window_size: int = 15,
-                 outlier_sensitivity: int = 50,
+                 outlier_sensitivity: int = 60,
 
                  # --- Parameters for image resizing ---
                  width: int = 0):
@@ -198,7 +198,7 @@ def _analyze_and_select_frames(self, frame_paths: List[str]) -> List[Dict[str, A
                 self.batch_buffer
             )
         elif self.selection_method == "outlier-removal":
-            # Outlier removal returns all frames with a 'selected' flag
+            # Outlier detection returns all frames with a 'selected' flag
             all_frames_data = select_outlier_removal_frames(
                 frames_with_scores,
                 self.outlier_window_size,

sharp_frames/ui/screens/configuration.py

@@ -351,7 +351,7 @@ def action_help(self) -> None:
 ## Selection Process
 
 After configuration, frames will be extracted and analyzed. You'll then see an interactive selection screen where you can:
-- Choose selection method (Best N, Batched, Outlier Removal)
+- Choose selection method (Best N, Batched, Outlier Detection)
 - Adjust parameters with real-time preview
 - See exactly how many frames will be selected
 

sharp_frames/ui/screens/configuration_original.py

@@ -246,7 +246,7 @@ def _create_selection_method_step(self, container) -> None:
         select_widget = Select([
             ("Best N frames - Choose a specific number of frames", "best-n"),
             ("Batched selection - Best frame from each batch", "batched"),
-            ("Outlier removal - Remove the blurriest frames", "outlier-removal")
+            ("Outlier detection - Reject local sharpness outliers", "outlier-removal")
         ], value=self.config_data.get("selection_method", "best-n"), id="selection-method-field")
         container.mount(select_widget)
 
@@ -306,17 +306,17 @@ def _create_method_params_step(self, container) -> None:
             input1.focus()
 
         elif method == "outlier-removal":
-            container.mount(Label("Outlier Removal Configuration:", classes="question"))
+            container.mount(Label("Outlier Detection Configuration:", classes="question"))
             container.mount(Label("Window size for comparison:"))
             input1 = Input(
                 value=str(self.config_data.get("outlier_window_size", 15)),
-                validators=[IntRangeValidator(min_value=3, max_value=30)],
+                validators=[IntRangeValidator(min_value=5, max_value=31)],
                 id="param1"
             )
             container.mount(input1)
             container.mount(Label("Sensitivity (0-100, higher = more aggressive):"))
             input2 = Input(
-                value=str(self.config_data.get("outlier_sensitivity", 50)),
+                value=str(self.config_data.get("outlier_sensitivity", 60)),
                 validators=[IntRangeValidator(min_value=0, max_value=100)],
                 id="param2"
             )
@@ -387,7 +387,7 @@ def _build_config_summary(self) -> str:
             lines.append(f"  Batch buffer: {self.config_data.get('batch_buffer', 2)}")
         elif method == "outlier-removal":
             lines.append(f"  Window size: {self.config_data.get('outlier_window_size', 15)}")
-            lines.append(f"  Sensitivity: {self.config_data.get('outlier_sensitivity', 50)}")
+            lines.append(f"  Sensitivity: {self.config_data.get('outlier_sensitivity', 60)}")
 
         # Only show output format and resize options for non-directory modes
         input_type = self.config_data.get("input_type", InputTypes.VIDEO)
@@ -559,8 +559,8 @@ def _save_current_step(self) -> bool:
                     elif method == "outlier-removal":
                         window_size = int(value1)
                         sensitivity = int(value2)
-                        if window_size < 3:
-                            self._show_error(step_container, "Window size must be at least 3")
+                        if window_size < 5:
+                            self._show_error(step_container, "Window size must be at least 5")
                             return False
                         if sensitivity < 0 or sensitivity > 100:
                             self._show_error(step_container, "Sensitivity must be between 0 and 100")
@@ -691,7 +691,7 @@ def _prepare_final_config(self) -> Dict[str, Any]:
         config["batch_size"] = 5
         config["batch_buffer"] = 2
         config["outlier_window_size"] = 15
-        config["outlier_sensitivity"] = 50
+        config["outlier_sensitivity"] = 60
 
         # Override with method-specific values
         if selection_method == "best-n":
@@ -702,6 +702,6 @@ def _prepare_final_config(self) -> Dict[str, Any]:
             config["batch_buffer"] = self.config_data.get("batch_buffer", 2)
         elif selection_method == "outlier-removal":
             config["outlier_window_size"] = self.config_data.get("outlier_window_size", 15)
-            config["outlier_sensitivity"] = self.config_data.get("outlier_sensitivity", 50)
+            config["outlier_sensitivity"] = self.config_data.get("outlier_sensitivity", 60)
 
-        return config 
+        return config